authorgravatar for me@paperdave.netdave caruso <me@paperdave.net> 2024-05-16 17:46:34-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-05-16 17:46:34-07:00
log634a70376c0023f46a6aae6611ed672397eae557
treeda7f7daddfe6575ce2771b5e1253ffda833a0910
parent5a950c1982ddcab8e8d84ec27310eb1e0afdeb1f
signature Signed by PGP key B5690EEEBB952194

rewrite entire pipeline in zig, implement sema (#30)

97 files changed, 9897 insertions(+), 5537 deletions(-)

.github/logo.png deleted
Binary files a/.github/logo.png and /dev/null differ
.gitignore+10-2
......@@ -1,2 +1,10 @@
1target
2*.js
1# If you have OS-specific or editor-specific files to ignore,
2# such as *.swp or .DS_Store, put those in your global
3# ~/.gitignore and put this in your ~/.gitconfig:
4#
5# [core]
6# excludesfile = ~/.gitignore
7#
8
9zig-out
10zig-cache
Cargo.lock deleted-759
......@@ -1,759 +0,0 @@
1# This file is automatically @generated by Cargo.
2# It is not intended for manual editing.
3version = 3
4
5[[package]]
6name = "addr2line"
7version = "0.21.0"
8source = "registry+https://github.com/rust-lang/crates.io-index"
9checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
10dependencies = [
11 "gimli",
12]
13
14[[package]]
15name = "adler"
16version = "1.0.2"
17source = "registry+https://github.com/rust-lang/crates.io-index"
18checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
19
20[[package]]
21name = "anstream"
22version = "0.6.5"
23source = "registry+https://github.com/rust-lang/crates.io-index"
24checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6"
25dependencies = [
26 "anstyle",
27 "anstyle-parse",
28 "anstyle-query",
29 "anstyle-wincon",
30 "colorchoice",
31 "utf8parse",
32]
33
34[[package]]
35name = "anstyle"
36version = "1.0.4"
37source = "registry+https://github.com/rust-lang/crates.io-index"
38checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"
39
40[[package]]
41name = "anstyle-parse"
42version = "0.2.3"
43source = "registry+https://github.com/rust-lang/crates.io-index"
44checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c"
45dependencies = [
46 "utf8parse",
47]
48
49[[package]]
50name = "anstyle-query"
51version = "1.0.2"
52source = "registry+https://github.com/rust-lang/crates.io-index"
53checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648"
54dependencies = [
55 "windows-sys",
56]
57
58[[package]]
59name = "anstyle-wincon"
60version = "3.0.2"
61source = "registry+https://github.com/rust-lang/crates.io-index"
62checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7"
63dependencies = [
64 "anstyle",
65 "windows-sys",
66]
67
68[[package]]
69name = "ast"
70version = "0.1.0"
71dependencies = [
72 "log",
73 "miette",
74 "num",
75 "thiserror",
76 "tokenizer",
77]
78
79[[package]]
80name = "autocfg"
81version = "1.1.0"
82source = "registry+https://github.com/rust-lang/crates.io-index"
83checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
84
85[[package]]
86name = "backtrace"
87version = "0.3.69"
88source = "registry+https://github.com/rust-lang/crates.io-index"
89checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837"
90dependencies = [
91 "addr2line",
92 "cc",
93 "cfg-if",
94 "libc",
95 "miniz_oxide",
96 "object",
97 "rustc-demangle",
98]
99
100[[package]]
101name = "backtrace-ext"
102version = "0.2.1"
103source = "registry+https://github.com/rust-lang/crates.io-index"
104checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50"
105dependencies = [
106 "backtrace",
107]
108
109[[package]]
110name = "bitflags"
111version = "2.4.1"
112source = "registry+https://github.com/rust-lang/crates.io-index"
113checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
114
115[[package]]
116name = "cc"
117version = "1.0.83"
118source = "registry+https://github.com/rust-lang/crates.io-index"
119checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
120dependencies = [
121 "libc",
122]
123
124[[package]]
125name = "cfg-if"
126version = "1.0.0"
127source = "registry+https://github.com/rust-lang/crates.io-index"
128checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
129
130[[package]]
131name = "clap"
132version = "4.4.14"
133source = "registry+https://github.com/rust-lang/crates.io-index"
134checksum = "33e92c5c1a78c62968ec57dbc2440366a2d6e5a23faf829970ff1585dc6b18e2"
135dependencies = [
136 "clap_builder",
137 "clap_derive",
138]
139
140[[package]]
141name = "clap_builder"
142version = "4.4.14"
143source = "registry+https://github.com/rust-lang/crates.io-index"
144checksum = "f4323769dc8a61e2c39ad7dc26f6f2800524691a44d74fe3d1071a5c24db6370"
145dependencies = [
146 "anstream",
147 "anstyle",
148 "clap_lex",
149 "strsim",
150]
151
152[[package]]
153name = "clap_derive"
154version = "4.4.7"
155source = "registry+https://github.com/rust-lang/crates.io-index"
156checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442"
157dependencies = [
158 "heck",
159 "proc-macro2",
160 "quote",
161 "syn",
162]
163
164[[package]]
165name = "clap_lex"
166version = "0.6.0"
167source = "registry+https://github.com/rust-lang/crates.io-index"
168checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
169
170[[package]]
171name = "cli"
172version = "0.1.0"
173dependencies = [
174 "ast",
175 "clap",
176 "js_ast",
177 "js_printer",
178 "log",
179 "miette",
180 "parser",
181 "sema",
182 "thiserror",
183 "tokenizer",
184]
185
186[[package]]
187name = "colorchoice"
188version = "1.0.0"
189source = "registry+https://github.com/rust-lang/crates.io-index"
190checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
191
192[[package]]
193name = "errno"
194version = "0.3.8"
195source = "registry+https://github.com/rust-lang/crates.io-index"
196checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245"
197dependencies = [
198 "libc",
199 "windows-sys",
200]
201
202[[package]]
203name = "form_urlencoded"
204version = "1.2.1"
205source = "registry+https://github.com/rust-lang/crates.io-index"
206checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
207dependencies = [
208 "percent-encoding",
209]
210
211[[package]]
212name = "gimli"
213version = "0.28.1"
214source = "registry+https://github.com/rust-lang/crates.io-index"
215checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
216
217[[package]]
218name = "heck"
219version = "0.4.1"
220source = "registry+https://github.com/rust-lang/crates.io-index"
221checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
222
223[[package]]
224name = "hermit-abi"
225version = "0.3.3"
226source = "registry+https://github.com/rust-lang/crates.io-index"
227checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
228
229[[package]]
230name = "idna"
231version = "0.5.0"
232source = "registry+https://github.com/rust-lang/crates.io-index"
233checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
234dependencies = [
235 "unicode-bidi",
236 "unicode-normalization",
237]
238
239[[package]]
240name = "is-terminal"
241version = "0.4.10"
242source = "registry+https://github.com/rust-lang/crates.io-index"
243checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455"
244dependencies = [
245 "hermit-abi",
246 "rustix",
247 "windows-sys",
248]
249
250[[package]]
251name = "is_ci"
252version = "1.1.1"
253source = "registry+https://github.com/rust-lang/crates.io-index"
254checksum = "616cde7c720bb2bb5824a224687d8f77bfd38922027f01d825cd7453be5099fb"
255
256[[package]]
257name = "js_ast"
258version = "0.1.0"
259dependencies = [
260 "ast",
261 "log",
262 "miette",
263 "num",
264 "thiserror",
265]
266
267[[package]]
268name = "js_printer"
269version = "0.1.0"
270dependencies = [
271 "js_ast",
272 "miette",
273 "num",
274 "thiserror",
275]
276
277[[package]]
278name = "libc"
279version = "0.2.152"
280source = "registry+https://github.com/rust-lang/crates.io-index"
281checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7"
282
283[[package]]
284name = "linux-raw-sys"
285version = "0.4.12"
286source = "registry+https://github.com/rust-lang/crates.io-index"
287checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456"
288
289[[package]]
290name = "log"
291version = "0.0.0"
292dependencies = [
293 "miette",
294 "thiserror",
295 "url",
296]
297
298[[package]]
299name = "memchr"
300version = "2.7.1"
301source = "registry+https://github.com/rust-lang/crates.io-index"
302checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
303
304[[package]]
305name = "miette"
306version = "5.10.0"
307source = "registry+https://github.com/rust-lang/crates.io-index"
308checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e"
309dependencies = [
310 "backtrace",
311 "backtrace-ext",
312 "is-terminal",
313 "miette-derive",
314 "once_cell",
315 "owo-colors",
316 "supports-color",
317 "supports-hyperlinks",
318 "supports-unicode",
319 "terminal_size",
320 "textwrap",
321 "thiserror",
322 "unicode-width",
323]
324
325[[package]]
326name = "miette-derive"
327version = "5.10.0"
328source = "registry+https://github.com/rust-lang/crates.io-index"
329checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c"
330dependencies = [
331 "proc-macro2",
332 "quote",
333 "syn",
334]
335
336[[package]]
337name = "miniz_oxide"
338version = "0.7.1"
339source = "registry+https://github.com/rust-lang/crates.io-index"
340checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
341dependencies = [
342 "adler",
343]
344
345[[package]]
346name = "num"
347version = "0.4.1"
348source = "registry+https://github.com/rust-lang/crates.io-index"
349checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af"
350dependencies = [
351 "num-bigint",
352 "num-complex",
353 "num-integer",
354 "num-iter",
355 "num-rational",
356 "num-traits",
357]
358
359[[package]]
360name = "num-bigint"
361version = "0.4.4"
362source = "registry+https://github.com/rust-lang/crates.io-index"
363checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0"
364dependencies = [
365 "autocfg",
366 "num-integer",
367 "num-traits",
368]
369
370[[package]]
371name = "num-complex"
372version = "0.4.4"
373source = "registry+https://github.com/rust-lang/crates.io-index"
374checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214"
375dependencies = [
376 "num-traits",
377]
378
379[[package]]
380name = "num-integer"
381version = "0.1.45"
382source = "registry+https://github.com/rust-lang/crates.io-index"
383checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
384dependencies = [
385 "autocfg",
386 "num-traits",
387]
388
389[[package]]
390name = "num-iter"
391version = "0.1.43"
392source = "registry+https://github.com/rust-lang/crates.io-index"
393checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"
394dependencies = [
395 "autocfg",
396 "num-integer",
397 "num-traits",
398]
399
400[[package]]
401name = "num-rational"
402version = "0.4.1"
403source = "registry+https://github.com/rust-lang/crates.io-index"
404checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0"
405dependencies = [
406 "autocfg",
407 "num-bigint",
408 "num-integer",
409 "num-traits",
410]
411
412[[package]]
413name = "num-traits"
414version = "0.2.17"
415source = "registry+https://github.com/rust-lang/crates.io-index"
416checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
417dependencies = [
418 "autocfg",
419]
420
421[[package]]
422name = "object"
423version = "0.32.2"
424source = "registry+https://github.com/rust-lang/crates.io-index"
425checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
426dependencies = [
427 "memchr",
428]
429
430[[package]]
431name = "once_cell"
432version = "1.19.0"
433source = "registry+https://github.com/rust-lang/crates.io-index"
434checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
435
436[[package]]
437name = "owo-colors"
438version = "3.5.0"
439source = "registry+https://github.com/rust-lang/crates.io-index"
440checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
441
442[[package]]
443name = "parser"
444version = "0.1.0"
445dependencies = [
446 "ast",
447 "log",
448 "miette",
449 "num",
450 "thiserror",
451 "tokenizer",
452]
453
454[[package]]
455name = "percent-encoding"
456version = "2.3.1"
457source = "registry+https://github.com/rust-lang/crates.io-index"
458checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
459
460[[package]]
461name = "proc-macro2"
462version = "1.0.76"
463source = "registry+https://github.com/rust-lang/crates.io-index"
464checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c"
465dependencies = [
466 "unicode-ident",
467]
468
469[[package]]
470name = "quote"
471version = "1.0.35"
472source = "registry+https://github.com/rust-lang/crates.io-index"
473checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
474dependencies = [
475 "proc-macro2",
476]
477
478[[package]]
479name = "rustc-demangle"
480version = "0.1.23"
481source = "registry+https://github.com/rust-lang/crates.io-index"
482checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
483
484[[package]]
485name = "rustix"
486version = "0.38.28"
487source = "registry+https://github.com/rust-lang/crates.io-index"
488checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316"
489dependencies = [
490 "bitflags",
491 "errno",
492 "libc",
493 "linux-raw-sys",
494 "windows-sys",
495]
496
497[[package]]
498name = "sema"
499version = "0.1.0"
500dependencies = [
501 "ast",
502 "miette",
503 "num",
504 "thiserror",
505]
506
507[[package]]
508name = "smawk"
509version = "0.3.2"
510source = "registry+https://github.com/rust-lang/crates.io-index"
511checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c"
512
513[[package]]
514name = "strsim"
515version = "0.10.0"
516source = "registry+https://github.com/rust-lang/crates.io-index"
517checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
518
519[[package]]
520name = "supports-color"
521version = "2.1.0"
522source = "registry+https://github.com/rust-lang/crates.io-index"
523checksum = "d6398cde53adc3c4557306a96ce67b302968513830a77a95b2b17305d9719a89"
524dependencies = [
525 "is-terminal",
526 "is_ci",
527]
528
529[[package]]
530name = "supports-hyperlinks"
531version = "2.1.0"
532source = "registry+https://github.com/rust-lang/crates.io-index"
533checksum = "f84231692eb0d4d41e4cdd0cabfdd2e6cd9e255e65f80c9aa7c98dd502b4233d"
534dependencies = [
535 "is-terminal",
536]
537
538[[package]]
539name = "supports-unicode"
540version = "2.0.0"
541source = "registry+https://github.com/rust-lang/crates.io-index"
542checksum = "4b6c2cb240ab5dd21ed4906895ee23fe5a48acdbd15a3ce388e7b62a9b66baf7"
543dependencies = [
544 "is-terminal",
545]
546
547[[package]]
548name = "syn"
549version = "2.0.48"
550source = "registry+https://github.com/rust-lang/crates.io-index"
551checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f"
552dependencies = [
553 "proc-macro2",
554 "quote",
555 "unicode-ident",
556]
557
558[[package]]
559name = "terminal_size"
560version = "0.1.17"
561source = "registry+https://github.com/rust-lang/crates.io-index"
562checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df"
563dependencies = [
564 "libc",
565 "winapi",
566]
567
568[[package]]
569name = "textwrap"
570version = "0.15.2"
571source = "registry+https://github.com/rust-lang/crates.io-index"
572checksum = "b7b3e525a49ec206798b40326a44121291b530c963cfb01018f63e135bac543d"
573dependencies = [
574 "smawk",
575 "unicode-linebreak",
576 "unicode-width",
577]
578
579[[package]]
580name = "thiserror"
581version = "1.0.56"
582source = "registry+https://github.com/rust-lang/crates.io-index"
583checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad"
584dependencies = [
585 "thiserror-impl",
586]
587
588[[package]]
589name = "thiserror-impl"
590version = "1.0.56"
591source = "registry+https://github.com/rust-lang/crates.io-index"
592checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471"
593dependencies = [
594 "proc-macro2",
595 "quote",
596 "syn",
597]
598
599[[package]]
600name = "tinyvec"
601version = "1.6.0"
602source = "registry+https://github.com/rust-lang/crates.io-index"
603checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
604dependencies = [
605 "tinyvec_macros",
606]
607
608[[package]]
609name = "tinyvec_macros"
610version = "0.1.1"
611source = "registry+https://github.com/rust-lang/crates.io-index"
612checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
613
614[[package]]
615name = "tokenizer"
616version = "0.1.0"
617dependencies = [
618 "log",
619 "miette",
620 "thiserror",
621]
622
623[[package]]
624name = "unicode-bidi"
625version = "0.3.15"
626source = "registry+https://github.com/rust-lang/crates.io-index"
627checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
628
629[[package]]
630name = "unicode-ident"
631version = "1.0.12"
632source = "registry+https://github.com/rust-lang/crates.io-index"
633checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
634
635[[package]]
636name = "unicode-linebreak"
637version = "0.1.5"
638source = "registry+https://github.com/rust-lang/crates.io-index"
639checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f"
640
641[[package]]
642name = "unicode-normalization"
643version = "0.1.22"
644source = "registry+https://github.com/rust-lang/crates.io-index"
645checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921"
646dependencies = [
647 "tinyvec",
648]
649
650[[package]]
651name = "unicode-width"
652version = "0.1.11"
653source = "registry+https://github.com/rust-lang/crates.io-index"
654checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85"
655
656[[package]]
657name = "url"
658version = "2.5.0"
659source = "registry+https://github.com/rust-lang/crates.io-index"
660checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633"
661dependencies = [
662 "form_urlencoded",
663 "idna",
664 "percent-encoding",
665]
666
667[[package]]
668name = "utf8parse"
669version = "0.2.1"
670source = "registry+https://github.com/rust-lang/crates.io-index"
671checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
672
673[[package]]
674name = "winapi"
675version = "0.3.9"
676source = "registry+https://github.com/rust-lang/crates.io-index"
677checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
678dependencies = [
679 "winapi-i686-pc-windows-gnu",
680 "winapi-x86_64-pc-windows-gnu",
681]
682
683[[package]]
684name = "winapi-i686-pc-windows-gnu"
685version = "0.4.0"
686source = "registry+https://github.com/rust-lang/crates.io-index"
687checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
688
689[[package]]
690name = "winapi-x86_64-pc-windows-gnu"
691version = "0.4.0"
692source = "registry+https://github.com/rust-lang/crates.io-index"
693checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
694
695[[package]]
696name = "windows-sys"
697version = "0.52.0"
698source = "registry+https://github.com/rust-lang/crates.io-index"
699checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
700dependencies = [
701 "windows-targets",
702]
703
704[[package]]
705name = "windows-targets"
706version = "0.52.0"
707source = "registry+https://github.com/rust-lang/crates.io-index"
708checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd"
709dependencies = [
710 "windows_aarch64_gnullvm",
711 "windows_aarch64_msvc",
712 "windows_i686_gnu",
713 "windows_i686_msvc",
714 "windows_x86_64_gnu",
715 "windows_x86_64_gnullvm",
716 "windows_x86_64_msvc",
717]
718
719[[package]]
720name = "windows_aarch64_gnullvm"
721version = "0.52.0"
722source = "registry+https://github.com/rust-lang/crates.io-index"
723checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"
724
725[[package]]
726name = "windows_aarch64_msvc"
727version = "0.52.0"
728source = "registry+https://github.com/rust-lang/crates.io-index"
729checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"
730
731[[package]]
732name = "windows_i686_gnu"
733version = "0.52.0"
734source = "registry+https://github.com/rust-lang/crates.io-index"
735checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"
736
737[[package]]
738name = "windows_i686_msvc"
739version = "0.52.0"
740source = "registry+https://github.com/rust-lang/crates.io-index"
741checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"
742
743[[package]]
744name = "windows_x86_64_gnu"
745version = "0.52.0"
746source = "registry+https://github.com/rust-lang/crates.io-index"
747checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"
748
749[[package]]
750name = "windows_x86_64_gnullvm"
751version = "0.52.0"
752source = "registry+https://github.com/rust-lang/crates.io-index"
753checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"
754
755[[package]]
756name = "windows_x86_64_msvc"
757version = "0.52.0"
758source = "registry+https://github.com/rust-lang/crates.io-index"
759checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
Cargo.toml deleted-26
......@@ -1,26 +0,0 @@
1[workspace.package]
2version = "0.1.0"
3edition = "2021"
4license = "MIT"
5authors = ["Lino Le Van", "Dave Caruso"]
6repository = "https://github.com/lino-levan/blight"
7
8[workspace]
9resolver = "2"
10members = [
11 "crates/ast",
12 "crates/cli",
13 "crates/js_ast",
14 "crates/js_printer",
15 "crates/log",
16 "crates/parser",
17 "crates/sema",
18 "crates/tokenizer",
19]
20
21[workspace.dependencies]
22clap = { version = "4.4.12", features = ["derive"] }
23miette = { version = "5.10.0", features = ["fancy"] }
24thiserror = "1.0.53"
25num = "0.4.1"
26url = "2.2.2"
README.md+70-10
......@@ -1,19 +1,79 @@
1<img src="/.github/logo.png" width="256">
1# Compiler
22
3# Blight
3This language is a mix of features from JavaScript, Rust, and Zig. Some notable features carried over from each include:
44
5Blight sure is a programming language.
5- **Rust** - Expressive control flow that is easy to write.
6- **Zig** - Execute any code at compile time, everything is an expression.
7- **JavaScript** - Garbage collected values with a versitile type system.
68
7## Developing
9## Standard Library
810
9Compiling a file:
11Currently, there is no standard library. One will be added later.
1012
11```bash
12cargo run compile --stdout ./examples/basics/fib.blight
13## Hello World
14
15```
16let print = @js("console").log;
17print("Hello, world!");
18```
19
20The `print` function is implemented by getting an external reference to the `console` JavaScript global, and then retriving the `log` property from it.
21
22## Primative Data Types
23
24| name | ts equivilent | description |
25| ---------- | --------------------- | ---------------------------------------------- |
26| `int` | `number` | a number in signed 32-bit range |
27| `float` | `number` | a 64-bit floating point number |
28| `bool` | `boolean` | either `true` or `false` |
29| `string` | `string` | a sequence of text characters |
30| `void` | `undefined` or `null` | detectable lack of a value |
31| `noreturn` | `never` | control flow prevents this value from existing |
32
33> **note**: `bigint` is planned but not added
34
35## Variables
36
37There are two types of variables, `let` and `var`.
38
39`let` bindings cannot be overridden, and should be thought of as aliases of the expression within. Referring to these identifiers is equivilent to inlining it's where it was used, aside from side effect duplication.
40
41```
42let hello: int = 1;
43let world = hello + 2;
44let hello = hello * world;
45
46@js("console").log(hello);
1347```
1448
15Running the compiler on a file:
49`var` bindings create mutable memory locations at runtime. This means they can be re-assigned to in arbitrary ways.
1650
17```bash
18cargo run run ./examples/basics/fib.blight
51```
52var message = "first message";
53
54if @js("Math").random() > 0.5 {
55 message = "second message"
56}
57
58@js("console").log(message);
59```
60
61`let` is preferred whenever possible, as it will lead to more readable code, and the compiler is able to optimize it better. The above can be rewritten as:
62
63```
64let message = if @js("Math").random() > 0.5 {
65 "second message"
66} else {
67 "first message"
68};
69
70@js("console").log(message);
71```
72
73The initializer for a `let` and `var` are always required, but the type annotations can be omitted when the data type can be inferred. For `let`, this is always the case. `var`, being runtime-mutable memory, only supports runtime representable data types. This means for numerical literals, an explicit type annotation is needed to determine weather to store an integer or a floating point number.
74
75```
76var a = 5; // error
77var b: int = 5; // ok
78var c: float = 5; // ok
1979```
build.zig created+96
......@@ -0,0 +1,96 @@
1const std = @import("std");
2
3pub fn build(b: *std.Build) void {
4 b.reference_trace = 20;
5
6 const target = b.standardTargetOptions(.{});
7 const optimize = b.standardOptimizeOption(.{});
8
9 const dep_opts = .{ .target = target, .optimize = optimize };
10
11 const zg = b.dependency("zg", dep_opts);
12 const tracer = b.dependency("tracer", dep_opts);
13 // const ada = b.dependency("ada", dep_opts);
14
15 const exe = b.addExecutable(.{
16 .name = "compiler",
17 .root_source_file = b.path("src/compiler/main.zig"),
18 .target = target,
19 .optimize = optimize,
20 });
21 b.installArtifact(exe);
22
23 exe.root_module.addImport("zg:PropsData", zg.module("PropsData"));
24 exe.root_module.addImport("tracer", tracer.module("tracer"));
25 // if (!target.result.isWasm()) {
26 // exe.root_module.addImport("ada", ada.module("ada"));
27 // }
28
29 {
30 const run_cmd = b.addRunArtifact(exe);
31 run_cmd.step.dependOn(b.getInstallStep());
32 if (b.args) |args| {
33 run_cmd.addArgs(args);
34 }
35 const run_step = b.step("compiler", "Run compiler CLI");
36 run_step.dependOn(&run_cmd.step);
37 }
38
39 // const wasm_api = b.addExecutable(.{
40 // .name = "compiler",
41 // .root_source_file = b.path("src/wasm_api.zig"),
42 // .target = b.resolveTargetQuery(.{
43 // .cpu_arch = .wasm32,
44 // .os_tag = .freestanding,
45 // }),
46 // .optimize = optimize,
47 // });
48 // wasm_api.root_module.export_symbol_names = &[_][]const u8{
49 // "iterateTokens",
50 // };
51 // wasm_api.entry = .disabled;
52 // const wasm_step = b.step("wasm", "Build wasm api");
53 // b.installArtifact(wasm_api);
54 // wasm_step.dependOn(&wasm_api.step);
55 // wasm_api.root_module.addImport("zg:PropsData", zg.module("PropsData"));
56
57 const release = b.step("release", "Package releases");
58 inline for (.{
59 b.resolveTargetQuery(.{ .cpu_arch = .x86_64, .os_tag = .linux, .abi = .musl }),
60 b.resolveTargetQuery(.{ .cpu_arch = .aarch64, .os_tag = .linux, .abi = .musl }),
61 b.resolveTargetQuery(.{ .cpu_arch = .x86_64, .os_tag = .macos }),
62 b.resolveTargetQuery(.{ .cpu_arch = .aarch64, .os_tag = .macos }),
63 b.resolveTargetQuery(.{ .cpu_arch = .x86_64, .os_tag = .windows }),
64 b.resolveTargetQuery(.{ .cpu_arch = .aarch64, .os_tag = .windows }),
65 }) |cross_target| {
66 const name = std.fmt.allocPrint(
67 b.allocator,
68 "{s}-{s}",
69 .{
70 @tagName(cross_target.result.cpu.arch),
71 @tagName(cross_target.result.os.tag),
72 },
73 ) catch @panic("OOM");
74
75 const cross_exe = b.addExecutable(.{
76 .name = "compiler",
77 .root_source_file = b.path("src/compiler/main.zig"),
78 .target = cross_target,
79 .optimize = .ReleaseSafe,
80 .strip = true,
81 });
82 cross_exe.root_module.addImport("zg:PropsData", zg.module("PropsData"));
83 cross_exe.root_module.addImport("tracer", tracer.module("tracer"));
84
85 release.dependOn(&b.addInstallFile(
86 cross_exe.getEmittedBin(),
87 b.pathJoin(&.{
88 name,
89 std.mem.concat(b.allocator, u8, &.{
90 "compiler",
91 cross_target.result.exeFileExt(),
92 }) catch @panic("OOM"),
93 }),
94 ).step);
95 }
96}
build.zig.zon created+22
......@@ -0,0 +1,22 @@
1.{
2 .name = "",
3 .version = "0.0.0",
4
5 .minimum_zig_version = "0.12.0",
6
7 .dependencies = .{
8 .zg = .{
9 .url = "https://codeberg.org/dude_the_builder/zg/archive/16735685fcc3410de361ba3411788ad1fb4fe188.tar.gz",
10 .hash = "1220fe9ac5cdb41833d327a78745614e67d472469f8666567bd8cf9f5847a52b1c51",
11 },
12 .tracer = .{
13 .url = "https://github.com/paperdave/zig-tracer/archive/d4c2eff040e9e69b9f08d2175b315f3055ad7ba3.tar.gz",
14 .hash = "12202d60766a46ceb4074ce5651a58a11f2f6b32927476a55c6889805073e8e3d261",
15 },
16 },
17 .paths = .{
18 "build.zig",
19 "build.zig.zon",
20 "src",
21 },
22}
crates/ast/Cargo.toml deleted-15
......@@ -1,15 +0,0 @@
1[package]
2name = "ast"
3version = "0.1.0"
4edition = "2021"
5
6[lib]
7path = "./lib.rs"
8
9[dependencies]
10miette.workspace = true
11thiserror.workspace = true
12num.workspace = true
13
14tokenizer = { path = "../tokenizer" } # Do not depend on the tokenizer
15log = { path = "../log" }
crates/ast/ast.rs deleted-211
......@@ -1,211 +0,0 @@
1use log::Range;
2
3use crate::types::{Literal, Type};
4use crate::{BinaryOperator, UnaryOperator};
5
6#[derive(Debug, Clone)]
7pub enum BuiltinId {
8 Raw,
9}
10
11#[derive(Debug, Clone)]
12pub struct Module {
13 pub comment: Option<DocComment>,
14 pub statements: Vec<Statement>,
15}
16
17#[derive(Debug, Clone)]
18pub struct Block {
19 pub statements: Vec<Statement>,
20 pub has_implicit_return: bool,
21}
22
23#[derive(Debug, Clone)]
24pub struct Param {
25 pub name: String,
26 pub param_type: Type,
27}
28
29#[derive(Debug, Clone)]
30
31pub struct VarDecl {
32 pub name: String,
33 pub data_type: Type,
34 pub init: Expr,
35 pub is_constant: bool,
36 pub is_pub: bool,
37 pub range: Range,
38 pub comment: Option<DocComment>,
39}
40
41#[derive(Debug, Clone)]
42pub struct FnDecl {
43 pub name: String,
44 pub return_type: Type,
45 pub params: Vec<Param>,
46 pub body: Expr,
47 pub is_pub: bool,
48 pub range: Range,
49 pub comment: Option<DocComment>,
50}
51
52#[derive(Debug, Clone)]
53pub struct Expr {
54 pub data: ExprData,
55 pub range: Range,
56 pub comment: Option<Comment>,
57}
58
59impl Expr {
60 pub fn new(data: ExprData, range: Range, comment: Option<Comment>) -> Self {
61 Self {
62 data,
63 range,
64 comment,
65 }
66 }
67}
68
69#[derive(Debug, Clone)]
70pub struct DocComment {
71 pub data: String,
72 pub range: Range,
73}
74
75#[derive(Debug, PartialEq, Clone)]
76pub struct Comment {
77 pub data: String,
78}
79
80impl Comment {
81 pub fn format(&self) -> String {
82 let mut lines = self.data.lines();
83 let first_line = lines.next().unwrap();
84 let mut result = format!("// {}", first_line);
85 for line in lines {
86 result.push_str(&format!("\n// {}", line));
87 }
88 result
89 }
90
91 pub fn to_doc_comment(self, range: Range) -> DocComment {
92 DocComment {
93 data: self.data,
94 range,
95 }
96 }
97}
98
99impl Into<Comment> for DocComment {
100 fn into(self) -> Comment {
101 Comment { data: self.data }
102 }
103}
104
105impl DocComment {
106 pub fn format(&self) -> String {
107 let mut lines = self.data.lines();
108 let first_line = lines.next().unwrap();
109 let mut result = format!("/// {}", first_line);
110 for line in lines {
111 result.push_str(&format!("\n/// {}", line));
112 }
113 result
114 }
115}
116
117#[derive(Debug, Clone)]
118pub struct BlockWithRange {
119 pub block: Block,
120 pub range: Range,
121}
122
123#[derive(Debug, Clone)]
124pub struct If {
125 pub condition: Box<Expr>,
126 pub yes: BlockWithRange,
127 pub no: Option<BlockOrIf>,
128}
129
130#[derive(Debug, Clone)]
131pub enum BlockOrIf {
132 Block(BlockWithRange),
133 If(Box<If>),
134}
135
136#[derive(Debug, Clone)]
137pub enum ExprData {
138 Block(Block),
139 Binary {
140 op: BinaryOperator,
141 left: Box<Expr>,
142 right: Box<Expr>,
143 },
144 Unary {
145 op: UnaryOperator,
146 value: Box<Expr>,
147 },
148 Fn {
149 name: String,
150 params: Vec<Param>,
151 return_type: Type,
152 body: Box<Expr>,
153 },
154 Call {
155 base: Box<Expr>,
156 arguments: Vec<Expr>,
157 },
158 Field {
159 base: Box<Expr>,
160 field: Box<Expr>,
161 },
162 Builtin {
163 id: BuiltinId,
164 arguments: Vec<Expr>,
165 },
166 If(If),
167 Return(Option<Box<Expr>>),
168 Literal(Literal),
169 Identifier(String),
170}
171
172#[derive(Debug, Clone)]
173pub enum Statement {
174 Expr(Expr),
175 VarDecl(VarDecl),
176 FnDecl(FnDecl),
177 Comment(Comment),
178}
179
180impl BuiltinId {
181 pub fn display_name(&self) -> &'static str {
182 match self {
183 Self::Raw => "@raw",
184 }
185 }
186
187 pub fn from_str(name: &str) -> Option<Self> {
188 match name {
189 "@raw" => Some(Self::Raw),
190 _ => None,
191 }
192 }
193}
194
195impl Statement {
196 /// This is stupid function im sorry
197 pub fn require_semicolon(&self, semicolon_hint: bool) -> bool {
198 match &self {
199 Statement::Expr(_) => !semicolon_hint,
200 Statement::VarDecl(_) => true,
201 _ => false,
202 }
203 }
204
205 pub fn is_expression(&self) -> bool {
206 match &self {
207 Statement::Expr(_) => true,
208 _ => false,
209 }
210 }
211}
crates/ast/lib.rs deleted-7
......@@ -1,7 +0,0 @@
1mod ast;
2mod operators;
3mod types;
4
5pub use ast::*;
6pub use operators::*;
7pub use types::*;
crates/ast/operators.rs deleted-316
......@@ -1,316 +0,0 @@
1use tokenizer::TokenType;
2
3use crate::Literal;
4
5#[derive(Debug, Clone, PartialEq)]
6pub enum BinaryOperator {
7 Add,
8 Sub,
9 Mul,
10 Div,
11 Pow,
12 Eq,
13 Gt,
14 Lt,
15 Gte,
16 Lte,
17 Lshift,
18 Rshift,
19 RshiftUnsigned,
20 BitAnd,
21 BitOr,
22 BitXor,
23 Mod,
24 NotEq,
25 And,
26 Or,
27
28 Assign,
29 AssignAdd,
30 AssignSub,
31 AssignMul,
32 AssignDiv,
33 AssignPow,
34 AssignLshift,
35 AssignRshift,
36 AssignRshiftUnsigned,
37 AssignBitAnd,
38 AssignBitOr,
39 AssignBitXor,
40 AssignMod,
41 AssignAnd,
42 AssignOr,
43}
44
45#[derive(Debug, Clone, PartialEq)]
46pub enum UnaryOperator {
47 Neg,
48 Not,
49 BitNot,
50}
51
52impl BinaryOperator {
53 pub fn to_str(&self) -> &'static str {
54 match self {
55 Self::Add => "+",
56 Self::Sub => "-",
57 Self::Div => "/",
58 Self::Mul => "*",
59 Self::Pow => "**",
60 Self::Eq => "==",
61 Self::Gt => ">",
62 Self::Lt => "<",
63 Self::Gte => ">=",
64 Self::Lte => "<=",
65 Self::Lshift => "<<",
66 Self::Rshift => ">>",
67 Self::RshiftUnsigned => ">>>",
68 Self::BitAnd => "&",
69 Self::BitOr => "|",
70 Self::BitXor => "^",
71 Self::Mod => "%",
72 Self::NotEq => "!=",
73 Self::And => "&&",
74 Self::Or => "||",
75 Self::AssignAdd => "+=",
76 Self::AssignSub => "-=",
77 Self::AssignDiv => "/=",
78 Self::AssignMul => "*=",
79 Self::AssignPow => "**=",
80 Self::Assign => "=",
81 Self::AssignLshift => "<<=",
82 Self::AssignRshift => ">>=",
83 Self::AssignRshiftUnsigned => ">>>=",
84 Self::AssignBitAnd => "&=",
85 Self::AssignBitOr => "|=",
86 Self::AssignBitXor => "^=",
87 Self::AssignMod => "%=",
88 Self::AssignAnd => "&&=",
89 Self::AssignOr => "||=",
90 }
91 }
92
93 pub fn to_string(&self) -> String {
94 String::from(self.to_str())
95 }
96
97 pub fn from_token(token: &TokenType) -> Option<BinaryOperator> {
98 match &token {
99 TokenType::And => Some(Self::BitAnd),
100 TokenType::AndAnd => Some(Self::And),
101 TokenType::BangEqual => Some(Self::NotEq),
102 TokenType::Caret => Some(Self::BitXor),
103 TokenType::EqualEqual => Some(Self::Eq),
104 TokenType::Gt => Some(Self::Gt),
105 TokenType::GtEqual => Some(Self::Gte),
106 TokenType::GtGt => Some(Self::Rshift),
107 TokenType::GtGtGt => Some(Self::RshiftUnsigned),
108 TokenType::Lt => Some(Self::Lt),
109 TokenType::LtEqual => Some(Self::Lte),
110 TokenType::LtLt => Some(Self::Lshift),
111 TokenType::Minus => Some(Self::Sub),
112 TokenType::Or => Some(Self::BitOr),
113 TokenType::OrOr => Some(Self::Or),
114 TokenType::Percent => Some(Self::Mod),
115 TokenType::Plus => Some(Self::Add),
116 TokenType::Slash => Some(Self::Div),
117 TokenType::Star => Some(Self::Mul),
118 TokenType::StarStar => Some(Self::Pow),
119 _ => None,
120 }
121 }
122
123 pub fn level(&self) -> OperatorLevel {
124 OperatorLevel::from(self)
125 }
126
127 /// Error case of this means 'types do not allow this operation'
128 pub fn perform(&self, left: Literal, right: Literal) -> Result<Literal, ()> {
129 match self {
130 BinaryOperator::Eq => Ok(Literal::Bool(left == right)),
131 BinaryOperator::NotEq => Ok(Literal::Bool(left != right)),
132
133 BinaryOperator::Or | BinaryOperator::And => match (left, right) {
134 (Literal::Bool(a), Literal::Bool(b)) => Ok(Literal::Bool(match self {
135 BinaryOperator::Or => a || b,
136 BinaryOperator::And => a && b,
137 _ => unreachable!(),
138 })),
139 _ => Err(()),
140 },
141
142 BinaryOperator::Add
143 | BinaryOperator::Sub
144 | BinaryOperator::Mul
145 | BinaryOperator::Div
146 | BinaryOperator::Pow
147 | BinaryOperator::Gt
148 | BinaryOperator::Lt
149 | BinaryOperator::Gte
150 | BinaryOperator::Lte
151 | BinaryOperator::Lshift
152 | BinaryOperator::Rshift
153 | BinaryOperator::RshiftUnsigned
154 | BinaryOperator::BitAnd
155 | BinaryOperator::BitOr
156 | BinaryOperator::BitXor
157 | BinaryOperator::Mod => match (left, right) {
158 (Literal::Number(a), Literal::Number(b)) => Ok(match self {
159 BinaryOperator::Add => Literal::Number(a + b),
160 BinaryOperator::Sub => Literal::Number(a - b),
161 BinaryOperator::Mul => Literal::Number(a * b),
162 BinaryOperator::Div => Literal::Number(a / b),
163 BinaryOperator::Pow => Literal::Number(a.powf(b)),
164 BinaryOperator::Gt => Literal::Bool(a > b),
165 BinaryOperator::Lt => Literal::Bool(a < b),
166 BinaryOperator::Gte => Literal::Bool(a >= b),
167 BinaryOperator::Lte => Literal::Bool(a <= b),
168 BinaryOperator::Lshift => Literal::Number(((a as i32) << (b as i32)).into()),
169 BinaryOperator::Rshift => Literal::Number(((a as i32) >> (b as i32)).into()),
170 BinaryOperator::RshiftUnsigned => {
171 Literal::Number(((a as u32) >> (b as u32)).into())
172 }
173 BinaryOperator::BitAnd => Literal::Number(((a as i32) & (b as i32)).into()),
174 BinaryOperator::BitOr => Literal::Number(((a as i32) | (b as i32)).into()),
175 BinaryOperator::BitXor => Literal::Number(((a as i32) ^ (b as i32)).into()),
176 BinaryOperator::Mod => Literal::Number(a % b),
177 _ => unreachable!(),
178 }),
179 _ => Err(()),
180 },
181
182 // Impossible condition doing assign-add to a literal
183 _ => Err(()),
184 }
185 }
186}
187
188impl UnaryOperator {
189 pub fn to_str(&self) -> &'static str {
190 match self {
191 Self::Neg => "-",
192 Self::Not => "!",
193 Self::BitNot => "~",
194 }
195 }
196
197 pub fn to_string(&self) -> String {
198 String::from(self.to_str())
199 }
200
201 pub fn perform(self, value: Literal) -> Result<Literal, ()> {
202 match self {
203 UnaryOperator::Neg => match value {
204 Literal::Number(a) => Ok(Literal::Number(-a)),
205 _ => Err(()),
206 },
207 UnaryOperator::Not => match value {
208 Literal::Bool(a) => Ok(Literal::Bool(!a)),
209 _ => Err(()),
210 },
211 UnaryOperator::BitNot => match value {
212 Literal::Number(a) => Ok(Literal::Number((!(a as i32)).into())),
213 _ => Err(()),
214 },
215 }
216 }
217}
218
219/// Taken from from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence#table
220///
221/// Blight doesn't have all of these operators, but the
222/// printer uses it to determine where to put parens.
223#[derive(PartialEq, PartialOrd, Debug)]
224pub enum OperatorLevel {
225 Accessor, // e. e[] e()
226 New, // new
227 Postfix, // expr++ expr--
228 Prefix, // ++e, --e, !e, ~e, +e, -e, typeof, void, delete
229
230 // these are the binary operators
231 Exponentiation, // **
232 Multiplicative, // *, /, %
233 Additive, // +, -
234 BitwiseShift, // <<, >>, >>>
235 Relational, // <, <=, >, >=
236 Equality, // ==, !=
237 BitwiseAnd, // &
238 BitwiseXor, // ^
239 BitwiseOr, // |
240 LogicalAnd, // &&
241 LogicalOr, // ||, ??
242
243 AssignmentAndOther, // =, +=, -=, *=, /=, %=, / <<=, >>=, >>>=, &=, ^=, |=
244 Comma, // ,
245}
246
247#[derive(PartialEq)]
248pub enum Associativity {
249 LeftToRight,
250 RightToLeft,
251}
252
253impl OperatorLevel {
254 pub fn associativity(&self) -> Associativity {
255 use Associativity::*;
256 use OperatorLevel::*;
257
258 match self {
259 Accessor => LeftToRight,
260 Exponentiation => RightToLeft,
261 Multiplicative => LeftToRight,
262 Additive => LeftToRight,
263 BitwiseShift => LeftToRight,
264 Relational => LeftToRight,
265 Equality => LeftToRight,
266 BitwiseAnd => LeftToRight,
267 BitwiseXor => LeftToRight,
268 BitwiseOr => LeftToRight,
269 LogicalAnd => LeftToRight,
270 LogicalOr => LeftToRight,
271 AssignmentAndOther => RightToLeft,
272 Comma => LeftToRight,
273
274 // These do not make sense to have an associativity
275 // You will likely not run into this if you do
276 // BinaryOperator::level(o).associativity()
277 New | Postfix | Prefix => unreachable!("{:?} does not have an associativity", self),
278 }
279 }
280}
281
282impl From<&BinaryOperator> for OperatorLevel {
283 fn from(operator: &BinaryOperator) -> Self {
284 use BinaryOperator::*;
285 use OperatorLevel::*;
286
287 match operator {
288 Add | Sub => Additive,
289 Mul | Div | Mod => Multiplicative,
290 Pow => Exponentiation,
291 Gt | Lt | Gte | Lte => Relational,
292 Lshift | Rshift | RshiftUnsigned => BitwiseShift,
293 BitAnd => BitwiseAnd,
294 BitOr => BitwiseOr,
295 BitXor => BitwiseXor,
296 Eq | NotEq => Equality,
297 And => LogicalAnd,
298 Or => LogicalOr,
299
300 Assign | AssignAdd | AssignSub | AssignMul | AssignDiv | AssignPow | AssignLshift
301 | AssignRshift | AssignRshiftUnsigned | AssignBitAnd | AssignBitOr | AssignBitXor
302 | AssignMod | AssignAnd | AssignOr => AssignmentAndOther,
303 }
304 }
305}
306
307impl From<UnaryOperator> for OperatorLevel {
308 fn from(operator: UnaryOperator) -> Self {
309 use OperatorLevel::*;
310 use UnaryOperator::*;
311
312 match operator {
313 Neg | Not | BitNot => Prefix,
314 }
315 }
316}
crates/ast/types.rs deleted-134
......@@ -1,134 +0,0 @@
1use miette::{Diagnostic, Result};
2use thiserror::Error;
3
4/// A value that is known at compile-time
5#[derive(Debug, Clone, PartialEq)]
6pub enum Literal {
7 Number(f64),
8 Bool(bool),
9 String(String),
10 Void,
11}
12
13impl std::fmt::Display for Literal {
14 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
15 match self {
16 Literal::Number(value) => write!(f, "{}", value),
17 Literal::Bool(value) => write!(f, "{}", value),
18 Literal::String(value) => write!(f, "{}", value),
19 Literal::Void => write!(f, "void"),
20 }
21 }
22}
23
24#[derive(Debug, Clone, PartialEq)]
25pub enum Type {
26 /// Variables without an explicit type are assigned this until
27 /// the type inferencer figures out what this is. There is not a
28 /// keyword for this.
29 Infer,
30 /// The type assigned to literals. This allows integer literals
31 /// to coerce to integer or float depending on the context, but
32 /// prevents implicit casting of runtime values.
33 Literal(Literal),
34 /// Return value of `@raw`. All type checks are disabled.
35 Any,
36 /// The type of `void`. This is the return type of functions
37 /// that do not return a value.
38 Void,
39 /// noreturn is the type of 'return', aka a value that never exists.
40 /// unlike void, this includes functions that do not return at all.
41 NoReturn,
42 /// 32 signed integer
43 Int,
44 /// 64 signed JS float
45 Float,
46 /// Boolean
47 Bool,
48 /// JS BigInt
49 BigInt,
50 /// JS String
51 String,
52 /// A union of multiple possible types. Might be removed
53 Union(Vec<Type>),
54}
55
56pub trait TypeCoerceable {
57 fn to_type(&self) -> Option<Type>;
58}
59
60impl TypeCoerceable for Literal {
61 fn to_type(&self) -> Option<Type> {
62 Some(match self {
63 Literal::Number(_) => Type::Float,
64 Literal::Bool(_) => Type::Bool,
65 Literal::String(_) => Type::String,
66 Literal::Void => Type::Void,
67 })
68 }
69}
70
71impl TypeCoerceable for Type {
72 fn to_type(&self) -> Option<Type> {
73 Some(self.clone())
74 }
75}
76
77impl std::fmt::Display for Type {
78 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
79 match self {
80 Type::Infer => write!(f, "infered type"),
81 Type::Any => write!(f, "opaque js value"),
82 Type::Literal(value) => write!(f, "{}", value),
83 Type::Int => write!(f, "int"),
84 Type::Float => write!(f, "float"),
85 Type::BigInt => write!(f, "bigint"),
86 Type::Bool => write!(f, "bool"),
87 Type::String => write!(f, "string"),
88 Type::NoReturn => write!(f, "noreturn"),
89 Type::Void => write!(f, "void"),
90 Type::Union(of) => {
91 for (i, t) in of.iter().enumerate() {
92 if i != 0 {
93 write!(f, " | ")?;
94 }
95 write!(f, "{}", t)?;
96 }
97 Ok(())
98 }
99 }
100 }
101}
102
103pub fn merge_types(a: Type, b: Type) -> Result<Type> {
104 match a {
105 Type::Infer => return Ok(b),
106 Type::Any => return Ok(Type::Any),
107 _ => {}
108 }
109
110 match b {
111 Type::Infer => return Ok(a),
112 Type::Any => return Ok(Type::Any),
113 _ => {}
114 }
115
116 if a == b {
117 return Ok(a);
118 }
119
120 Err(TypeError::new(format!("TODO: Implement type merging for {} and {}", a, b)).into())
121}
122
123#[derive(Error, Diagnostic, Debug)]
124#[error("TypeError")]
125pub struct TypeError {
126 #[help]
127 error: String,
128}
129
130impl TypeError {
131 pub fn new(error: String) -> Self {
132 Self { error }
133 }
134}
crates/cli/Cargo.toml deleted-21
......@@ -1,21 +0,0 @@
1[package]
2name = "cli"
3version = "0.1.0"
4edition = "2021"
5
6[[bin]]
7name = "blight"
8path = "./main.rs"
9
10[dependencies]
11clap.workspace = true
12miette.workspace = true
13thiserror.workspace = true
14
15tokenizer = { path = "../tokenizer" }
16ast = { path = "../ast" }
17parser = { path = "../parser" }
18js_ast = { path = "../js_ast" }
19js_printer = { path = "../js_printer" }
20sema = { path = "../sema" }
21log = { path = "../log" }
crates/cli/main.rs deleted-172
......@@ -1,172 +0,0 @@
1use std::{io::Write, path::PathBuf};
2
3use clap::{Parser, Subcommand};
4use miette::{diagnostic, Result};
5
6use js_printer;
7use tokenizer;
8
9#[derive(Parser)]
10#[command(author, version, about, long_about = None)]
11struct Cli {
12 /// Turn debugging information on
13 #[arg(short, long, action = clap::ArgAction::Count)]
14 debug: u8,
15
16 #[command(subcommand)]
17 command: Commands,
18}
19
20#[derive(Subcommand)]
21enum Commands {
22 /// Compiles a file into javascript
23 Compile {
24 file: PathBuf,
25 // Whether to output to stdout instead of a file
26 #[arg(long, default_value_t = false)]
27 stdout: bool,
28 /// What javascript runtime to compile to
29 #[arg(short, long, default_value = "deno")]
30 target: String,
31 /// Format the output javascript?
32 #[arg(long, default_value_t = true)]
33 format: bool,
34 },
35 Run {
36 file: PathBuf,
37 /// What javascript runtime to compile to and use
38 #[arg(short, long, default_value = "deno")]
39 target: String,
40 },
41 Astdump {
42 file: PathBuf,
43 },
44}
45
46fn get_runtime(runtime: &String) -> String {
47 match std::env::var("USER") {
48 Ok(user) if user == "dave" => String::from("bun"),
49 _ => runtime.clone(),
50 }
51}
52
53pub fn transpile(content: String, writer: &mut dyn Write) -> Result<()> {
54 // 1. Tokenize the input
55 let tokens = tokenizer::tokenize(content)?;
56 // 2. Parse the tokens into an AST
57 let mut blight_ast = parser::parse(tokens)?;
58 // 3. Semantic analysis.
59 sema::analyze_module(&mut blight_ast)?;
60 // 4. Optimize the AST
61 // optimize::optimize_module(&mut blight_ast);
62 // 5. Translate the AST into javascript
63 let js_ast = js_ast::blight_to_js(blight_ast);
64 // 6. Print the JS AST
65 // TODO(@paperdave): use std::io::Write instead of a string
66 js_printer::Print::print(&js_ast, writer)
67 .map_err(|e| diagnostic!("Error while printing: {}", e))?;
68 Ok(())
69}
70
71pub fn transpile_to_string(content: String) -> Result<String> {
72 let mut buffer = Vec::new();
73 transpile(content, &mut buffer)?;
74 Ok(String::from_utf8(buffer).unwrap())
75}
76
77pub fn astdump(content: String) -> Result<String> {
78 // 1. Tokenize the input
79 let tokens = tokenizer::tokenize(content)?;
80 // 2. Parse the tokens into an AST
81 let blight_ast = parser::parse(tokens)?;
82 return Ok(format!("{:#?}", blight_ast));
83}
84
85fn main() -> Result<()> {
86 let cli = Cli::parse();
87
88 match cli.command {
89 Commands::Compile {
90 file,
91 stdout,
92 target: _,
93 format,
94 } => {
95 let contents = match std::fs::read_to_string(file.clone()) {
96 Ok(contents) => contents,
97 Err(e) => {
98 eprintln!("Failed to read {}: {}", file.display(), e);
99 std::process::exit(1);
100 }
101 };
102
103 let mut result = transpile_to_string(contents.clone())?;
104
105 if format {
106 let mut formatter = std::process::Command::new("deno")
107 .stdin(std::process::Stdio::piped())
108 .stdout(std::process::Stdio::piped())
109 .stderr(std::process::Stdio::inherit())
110 .arg("fmt")
111 .arg("-")
112 .spawn()
113 .expect("Failed to execute 'deno fmt' on output");
114 let mut stdin = formatter.stdin.take().expect("Failed to open stdin");
115 stdin
116 .write_all(result.as_bytes())
117 .expect("Failed to write to stdin");
118 stdin.flush().expect("Failed to flush stdin");
119 drop(stdin);
120 let out = formatter
121 .wait_with_output()
122 .expect("Failed to execute 'deno fmt' on output");
123 result = String::from_utf8(out.stdout).unwrap();
124 }
125
126 if stdout {
127 println!("{}", result);
128 } else {
129 std::fs::write(file.with_extension("js"), result).unwrap();
130 }
131 }
132 Commands::Run { file, target } => {
133 let contents = std::fs::read_to_string(file.clone()).unwrap();
134 let result = transpile_to_string(contents.clone())?;
135 let mut temp_file = std::env::temp_dir();
136 temp_file.push("temp.js");
137 std::fs::write(temp_file.clone(), result).unwrap();
138
139 let rt = get_runtime(&target);
140
141 match rt.as_str() {
142 "deno" => std::process::Command::new("deno")
143 .arg("run")
144 .arg("-A")
145 .arg(temp_file)
146 .spawn()
147 .expect("Failed to execute process"),
148 "node" => std::process::Command::new("node")
149 .arg(temp_file)
150 .spawn()
151 .expect("Failed to execute process"),
152 "bun" => std::process::Command::new("bun")
153 .arg("run")
154 .arg(temp_file)
155 .spawn()
156 .expect("Failed to execute process"),
157 _ => {
158 panic!("Unknown runtime: {}", target);
159 }
160 }
161 .wait()
162 .unwrap();
163 }
164 Commands::Astdump { file } => {
165 let contents = std::fs::read_to_string(file.clone()).unwrap();
166 let result = astdump(contents.clone())?;
167 println!("{}", result);
168 }
169 }
170
171 Ok(())
172}
crates/js_ast/Cargo.toml deleted-15
......@@ -1,15 +0,0 @@
1[package]
2name = "js_ast"
3version = "0.1.0"
4edition = "2021"
5
6[lib]
7path = "./lib.rs"
8
9[dependencies]
10miette.workspace = true
11thiserror.workspace = true
12num.workspace = true
13
14ast = { path = "../ast" }
15log = { path = "../log" }
crates/js_ast/convert/block_convert_context.rs deleted-244
......@@ -1,244 +0,0 @@
1use crate::js_ast::*;
2use ast::*;
3use log::Range;
4use std::collections::{HashMap, HashSet};
5use std::vec;
6
7use super::tree_shake_statement_list;
8
9/// A block convert context keeps track of state while converting non-valid JS expressions to JS
10pub struct BlockConvertContext {
11 /// Statements in this block context
12 /// This may be the merged contents of multiple blocks
13 statements: Vec<JsStatement>,
14 ids: HashSet<String>,
15 next_id: u32,
16 scopes: Vec<VirtualScope>,
17 /// If the last item in `statements` is actually valid
18 has_last_statement: bool,
19}
20
21struct VirtualScope {
22 renames: HashMap<String, String>,
23}
24
25pub struct TempId {
26 unmapped_name: String,
27}
28
29pub struct StatementHandle {
30 index: Option<u32>,
31}
32
33impl StatementHandle {
34 pub fn delete(self, ctx: &mut BlockConvertContext) {
35 if let Some(index) = self.index {
36 ctx.statements.remove(index as usize);
37 }
38 }
39}
40
41impl TempId {
42 pub fn get(&self, hoist_context: &BlockConvertContext) -> String {
43 hoist_context.remap_identifier(&self.unmapped_name)
44 }
45}
46
47impl BlockConvertContext {
48 pub fn new() -> Self {
49 Self {
50 statements: Vec::new(),
51 ids: HashSet::new(),
52 next_id: 0,
53 scopes: vec![VirtualScope {
54 renames: HashMap::new(),
55 }],
56 has_last_statement: false,
57 }
58 }
59
60 pub fn new_child(&self) -> Self {
61 Self {
62 statements: Vec::new(),
63 ids: self.ids.clone(),
64 next_id: 0,
65 scopes: vec![VirtualScope {
66 renames: HashMap::new(),
67 }],
68 has_last_statement: false,
69 }
70 }
71
72 /// A virtual scope is a scope that exists in blight syntax
73 /// but may or may not exist in blight. With these it is important to
74 pub fn push_virtual_scope(&mut self) {
75 self.scopes.push(VirtualScope {
76 renames: HashMap::new(),
77 });
78 }
79
80 /// See `push_virtual_scope` for more information on what this is.
81 pub fn pop_virtual_scope(&mut self) {
82 self.scopes.pop();
83 debug_assert!(self.scopes.len() > 0);
84 }
85
86 pub fn generate_temp_id(&mut self, basename: &str) -> TempId {
87 self.next_id += 1;
88 loop {
89 let name = format!("{}{}", basename, self.next_id);
90 if !self.ids.contains(&name) {
91 self.ids.insert(name.clone());
92 return TempId {
93 unmapped_name: name,
94 };
95 }
96 self.next_id += 1;
97 }
98 }
99
100 /// This does some extra optimizations:
101 /// - `let a = b;` will not be emitted but instad write a variable alias
102 pub fn add_statement(&mut self, statement: JsStatement) -> StatementHandle {
103 match statement.data {
104 JsStatementData::Declaration(JSDeclaration {
105 name,
106 init:
107 Some(JsExpr {
108 data: JsExprData::Identifier(init_name),
109 ..
110 }),
111 is_constant: true,
112 }) => {
113 self.scopes
114 .last_mut()
115 .unwrap()
116 .renames
117 .insert(name, init_name);
118 self.has_last_statement = false;
119 return StatementHandle { index: None };
120 }
121 _ => {}
122 }
123 self.statements.push(statement);
124 self.has_last_statement = true;
125 StatementHandle {
126 index: Some(self.statements.len() as u32 - 1),
127 }
128 }
129
130 /// Get the last statement if it exists.
131 /// If the statement was optimized away, this returns None
132 pub fn last_statement(&self) -> Option<&JsStatement> {
133 if self.has_last_statement {
134 Some(self.statements.last().unwrap())
135 } else {
136 None
137 }
138 }
139
140 pub fn generate_id(&mut self, basename: &String) -> TempId {
141 let mut id = 0u32;
142 loop {
143 let name = match id {
144 0 => basename.clone(),
145 _ => format!("{}{}", basename, id),
146 };
147 if !self.ids.contains(&name) {
148 self.ids.insert(name.clone());
149 return TempId {
150 unmapped_name: name,
151 };
152 }
153 id += 1;
154 }
155 }
156
157 /// Given a blight identifier, remap it to a new identifier.
158 pub fn remap_identifier(&self, name: &String) -> String {
159 for scope in self.scopes.iter().rev() {
160 if let Some(remapped) = scope.renames.get(name) {
161 return remapped.clone();
162 }
163 }
164 name.clone()
165 }
166
167 /// Declare an identifier in the current scope.
168 pub fn declare_identifier(&mut self, name: String) -> TempId {
169 let remapped = self.generate_id(&name);
170 let scope = self.scopes.last_mut().unwrap();
171 self.ids.insert(remapped.unmapped_name.clone());
172 scope.renames.insert(name, remapped.unmapped_name.clone());
173 remapped
174 }
175
176 pub fn take_last_statement(&mut self) -> Option<JsStatement> {
177 if !self.has_last_statement {
178 return None;
179 }
180 self.has_last_statement = false;
181 Some(self.statements.pop().unwrap())
182 }
183
184 pub fn take_last_statement_if_expr(&mut self) -> Option<JsExpr> {
185 if !self.has_last_statement {
186 return None;
187 }
188 match self.statements.last().unwrap().data {
189 JsStatementData::Expression(_) => {
190 self.has_last_statement = false;
191 match self.statements.pop().unwrap().data {
192 JsStatementData::Expression(expr) => Some(expr),
193 _ => unreachable!(),
194 }
195 }
196 _ => return None,
197 }
198 }
199}
200
201pub fn new_js_assignment_statement(expr: JsExpr, target: String, range: Range) -> JsStatement {
202 JsStatement::new(
203 JsStatementData::Expression(new_js_assignment_expr(expr, target, range.clone())),
204 range,
205 None,
206 )
207}
208
209pub fn new_js_assignment_expr(expr: JsExpr, target: String, range: Range) -> JsExpr {
210 JsExpr::new(
211 JsExprData::Binary {
212 op: BinaryOperator::Assign,
213 left: Box::new(JsExpr::new(
214 JsExprData::Identifier(target),
215 range.clone(),
216 None,
217 )),
218 right: Box::new(expr),
219 },
220 range,
221 None,
222 )
223}
224
225impl Into<JsBlock> for BlockConvertContext {
226 fn into(self) -> JsBlock {
227 JsBlock::new(self.into(), Range::EMPTY)
228 }
229}
230
231impl Into<JsModule> for BlockConvertContext {
232 fn into(self) -> JsModule {
233 JsModule {
234 statements: self.into(),
235 }
236 }
237}
238
239impl Into<Vec<JsStatement>> for BlockConvertContext {
240 fn into(mut self) -> Vec<JsStatement> {
241 tree_shake_statement_list(&mut self.statements);
242 self.statements
243 }
244}
crates/js_ast/convert/convert_valid_block.rs deleted-37
......@@ -1,37 +0,0 @@
1use super::convert_valid_statement_to_expr;
2use super::BlockConvertContext;
3use super::IsValidJSExpression;
4use crate::js_ast::*;
5use ast::*;
6use log::Range;
7
8pub fn convert_valid_block(ctx: &mut BlockConvertContext, block: Block, range: Range) -> JsExpr {
9 debug_assert!(
10 block.is_valid_js_expression(),
11 "must call convert_valid_block with valid js expression"
12 );
13
14 let len = block.statements.len();
15 let range = range;
16 let block_expr = match len {
17 0 => JsExpr::new(JsExprData::Null, range.clone(), None),
18 // a block expression with a single expression is just that expression
19 1 => convert_valid_statement_to_expr(ctx, block.statements.into_iter().next().unwrap()),
20 // Since blocks are not valid expressions in JS, we create a
21 // comma expression from a block with more than one statement.
22 2.. => {
23 let mut items = Vec::new();
24 for statement in block.statements.into_iter() {
25 items.push(convert_valid_statement_to_expr(ctx, statement));
26 }
27 JsExpr::new(JsExprData::Comma { items }, range.clone(), None)
28 }
29 };
30
31 // a block without implicit return is to be type void, so we wrap in a void expression
32 if !block.has_implicit_return && len > 0 {
33 JsExpr::new(JsExprData::Void(Box::new(block_expr)), range, None)
34 } else {
35 block_expr
36 }
37}
crates/js_ast/convert/convert_valid_expr.rs deleted-115
......@@ -1,115 +0,0 @@
1use super::convert_valid_block;
2use super::BlockConvertContext;
3use crate::convert::convert_valid_if;
4use crate::convert::process_function::process_function_to_arrow;
5use crate::convert::IsValidJSExpression;
6use crate::js_ast::*;
7use ast::*;
8
9/// Translate a blight ast expression that is a valid JS expression, to a JS expression
10/// Precondition: it is a valid JS expression, check this with `is_valid_js_expression`
11pub fn convert_valid_expr(ctx: &mut BlockConvertContext, expr: Expr) -> JsExpr {
12 // Assert may be skipped in release builds as all failures are
13 // annotated with a release-checked panic
14 debug_assert!(
15 expr.is_valid_js_expression(),
16 "must call convert_valid_expr with valid js expression (use hoist instead)"
17 );
18
19 let range = expr.range;
20 let comment = expr.comment;
21
22 match expr.data {
23 ExprData::Identifier(str) => JsExpr::new(
24 JsExprData::Identifier(ctx.remap_identifier(&str)),
25 range,
26 comment,
27 ),
28
29 ExprData::Literal(Literal::Bool(value)) => {
30 JsExpr::new(JsExprData::Bool(value), range, comment)
31 }
32 ExprData::Literal(Literal::Number(value)) => {
33 JsExpr::new(JsExprData::Number(value), range, comment)
34 }
35 ExprData::Literal(Literal::String(value)) => {
36 JsExpr::new(JsExprData::String(value), range, comment)
37 }
38 ExprData::Literal(Literal::Void) => JsExpr::new(JsExprData::Null, range, comment),
39
40 ExprData::Block(block) => convert_valid_block(ctx, block, range),
41
42 ExprData::Binary { op, left, right } => {
43 let left = convert_valid_expr(ctx, *left);
44 let right = convert_valid_expr(ctx, *right);
45 JsExpr::new(
46 JsExprData::Binary {
47 op,
48 left: Box::new(left),
49 right: Box::new(right),
50 },
51 range,
52 comment,
53 )
54 }
55 ExprData::Unary { op, value } => {
56 let value = convert_valid_expr(ctx, *value);
57 JsExpr::new(
58 JsExprData::Unary {
59 op,
60 value: Box::new(value),
61 },
62 range,
63 comment,
64 )
65 }
66 ExprData::Fn { body, params, .. } => {
67 process_function_to_arrow(ctx, *body, params, range, comment)
68 }
69 ExprData::Builtin { id, arguments } => match id {
70 BuiltinId::Raw => {
71 let first_arg = arguments.iter().next().unwrap();
72 // TODO(@paperdave): this string clone is stupid
73 let str = match first_arg.data.clone() {
74 ExprData::Literal(Literal::String(s)) => s,
75 _ => panic!("raw expects a string"),
76 };
77 JsExpr::new(JsExprData::Raw(str), range, comment)
78 }
79 },
80 ExprData::If(expr) => convert_valid_if(ctx, expr, range, comment),
81
82 ExprData::Call { base, arguments } => {
83 let base = convert_valid_expr(ctx, *base);
84 let arguments = arguments
85 .into_iter()
86 .map(|x| convert_valid_expr(ctx, x))
87 .collect();
88 JsExpr::new(
89 JsExprData::Call {
90 base: Box::new(base),
91 arguments,
92 },
93 range,
94 comment,
95 )
96 }
97
98 ExprData::Field { base, field } => JsExpr::new(
99 match field.data {
100 ExprData::Literal(Literal::String(name)) => JsExprData::Dot {
101 base: Box::new(convert_valid_expr(ctx, *base)),
102 name,
103 },
104 _ => JsExprData::Index {
105 base: Box::new(convert_valid_expr(ctx, *base)),
106 index: Box::new(convert_valid_expr(ctx, *field)),
107 },
108 },
109 range,
110 comment,
111 ),
112
113 ExprData::Return(_) => unreachable!("convert_valid_expr called on return"),
114 }
115}
crates/js_ast/convert/convert_valid_if.rs deleted-38
......@@ -1,38 +0,0 @@
1use super::convert_valid_block;
2use super::convert_valid_expr;
3use super::BlockConvertContext;
4use crate::convert::IsValidJSExpression;
5use crate::js_ast::*;
6use ast::*;
7use log::Range;
8
9/// Convert a valid blight if statement to a JS expression
10pub fn convert_valid_if(
11 ctx: &mut BlockConvertContext,
12 expr: If,
13 range: Range,
14 comment: Option<Comment>,
15) -> JsExpr {
16 debug_assert!(
17 expr.is_valid_js_expression(),
18 "convert_valid_if on an invalid expression (use hoist instead)"
19 );
20 let condition = convert_valid_expr(ctx, *expr.condition);
21
22 let yes = convert_valid_block(ctx, expr.yes.block, expr.yes.range);
23 let no = match expr.no {
24 Some(BlockOrIf::Block(b)) => Some(convert_valid_block(ctx, b.block, b.range)),
25 Some(BlockOrIf::If(else_if)) => Some(convert_valid_if(ctx, *else_if, Range::EMPTY, None)),
26 None => None,
27 };
28
29 JsExpr::new(
30 JsExprData::Ternary {
31 condition: Box::new(condition),
32 yes: Box::new(yes),
33 no: no.map(Box::new),
34 },
35 range,
36 comment,
37 )
38}
crates/js_ast/convert/convert_valid_statement.rs deleted-32
......@@ -1,32 +0,0 @@
1use super::convert_valid_expr;
2use super::is_valid_js_expression::*;
3use super::BlockConvertContext;
4use crate::js_ast::*;
5use ast::*;
6
7/// Translate a blight ast statement that is a valid JS expression, to a JS expression
8/// Precondition: it is a valid JS expression, check this with `is_valid_js_expression`
9pub fn convert_valid_statement_to_expr(
10 ctx: &mut BlockConvertContext,
11 statement: Statement,
12) -> JsExpr {
13 debug_assert!(
14 statement.is_valid_js_expression(),
15 "convert_valid_statement_to_expr on an invalid expression (use hoist instead)"
16 );
17
18 match statement {
19 ast::Statement::Expr(expr) => convert_valid_expr(ctx, expr),
20 _ => unreachable!("attempt to convert statement to js without hoisting"),
21 }
22}
23
24/// Translate a blight ast statement that is a valid JS expression, to a JS statement
25/// Precondition: it is a valid JS expression, check this with `is_valid_js_expression`
26///
27/// TODO(@paperdave): Replace this with Into<JsStatement> for JsExpr
28pub fn convert_valid_statement(ctx: &mut BlockConvertContext, statement: Statement) -> JsStatement {
29 let expr = convert_valid_statement_to_expr(ctx, statement);
30 let range = expr.range.clone();
31 JsStatement::new(JsStatementData::Expression(expr), range, None)
32}
crates/js_ast/convert/hoist_block_to_expr.rs deleted-78
......@@ -1,78 +0,0 @@
1use super::is_valid_js_expression::*;
2use super::process_statement;
3use super::BlockConvertContext;
4use crate::js_ast::*;
5use ast::*;
6use log::Range;
7
8/// This block hoist is used when we have a block in an expression context.
9///
10/// All the expressions in the block are lifted above the currently
11/// processed expression, and then a temporary variable is returned.
12/// (or the last expression might get inlined as the return value)
13///
14/// A "virtual scope" is created for this block, which handles renaming
15/// situations like:
16///
17/// let x = 1;
18/// {
19/// let x = 1;
20/// }
21/// call(x)
22///
23/// The output JS of the above would be:
24///
25/// const x = 1;
26/// const x2 = 1;
27/// call(x)
28///
29/// For expressions that have branches, use `hoist_block_to_js_block`
30pub fn hoist_block_to_expr(
31 ctx: &mut BlockConvertContext,
32 block: Block,
33 range: Range,
34 comment: Option<Comment>,
35) -> Option<JsExpr> {
36 debug_assert!(
37 !block.is_valid_js_expression(),
38 "hoist_block on a valid expression, should call convert_valid_block instead"
39 );
40
41 let temporary = ctx.generate_temp_id("_block");
42
43 ctx.push_virtual_scope();
44
45 for statement in block.statements {
46 process_statement(ctx, statement);
47 }
48
49 ctx.pop_virtual_scope();
50
51 // If the last statement is an expression, we need
52 // to assign to the temporary variable.
53 let result = match ctx.last_statement().map(|x| &x.data) {
54 Some(JsStatementData::Expression(_)) if block.has_implicit_return => {
55 match ctx.take_last_statement().unwrap().data {
56 JsStatementData::Expression(expr) => expr,
57 _ => unreachable!(),
58 }
59 }
60 _ => JsExpr::new(JsExprData::Null, range.clone(), None),
61 };
62
63 ctx.add_statement(JsStatement::new(
64 JsStatementData::Declaration(JSDeclaration {
65 name: temporary.get(&ctx),
66 init: Some(result),
67 is_constant: false,
68 }),
69 range.clone(),
70 comment,
71 ));
72
73 Some(JsExpr::new(
74 JsExprData::Identifier(temporary.get(&ctx)),
75 range,
76 None,
77 ))
78}
crates/js_ast/convert/hoist_block_to_js_block.rs deleted-70
......@@ -1,70 +0,0 @@
1use crate::js_ast::*;
2use ast::*;
3use log::Range;
4
5use super::*;
6
7/// This block hoist is used when we actually need to create a JsBlock, which
8/// is needed if there is an invalid expression within a branch of code.
9///
10/// For example, `if` has two branches, the 'yes' and 'no' paths.
11/// When we cannot lower it as a ternary, we have to create two JsBlocks
12///
13/// The `target` param is a shared temporary variable that will be assigned
14/// to the result of the block.
15///
16/// Since blocks can be void or noreturn, this target is not always
17/// assigned to. The `did_assign` flag is used to indicate that.
18///
19/// For expressions that do not have branches, use `hoist_block_to_expr`
20///
21/// TODO(@paperdave): move `did_assign` into the return value
22pub fn hoist_block_to_js_block(
23 ctx: &mut BlockConvertContext,
24 block: Block,
25 range: Range,
26 target: &TempId,
27 did_assign: &mut bool,
28) -> JsBlock {
29 // A non-block will become a block with a single expression:
30 // _temp = expr;
31 //
32 // We will also apply this transformation to blocks that are valid
33 // expressions, because that is more concise.
34 if block.is_valid_js_expression() {
35 return JsBlock::new(
36 vec![new_js_assignment_statement(
37 convert_valid_block(ctx, block, range.clone()),
38 target.get(ctx),
39 range.clone(),
40 )],
41 range,
42 );
43 }
44
45 let mut child = ctx.new_child();
46
47 for statement in block.statements {
48 if let None = process_statement(&mut child, statement) {
49 return child.into();
50 };
51 }
52
53 // If the last statement is an expression, we need
54 // to assign to the temporary variable.
55 if block.has_implicit_return {
56 match child.take_last_statement_if_expr() {
57 Some(expr) => {
58 *did_assign = true;
59 child.add_statement(new_js_assignment_statement(
60 expr,
61 target.get(ctx),
62 range.clone(),
63 ));
64 }
65 _ => {}
66 }
67 }
68
69 child.into()
70}
crates/js_ast/convert/hoist_expr.rs deleted-139
......@@ -1,139 +0,0 @@
1use super::convert_valid_expr;
2use super::hoist_block_to_expr;
3use super::hoist_if_expr;
4use super::is_valid_js_expression::*;
5use super::BlockConvertContext;
6// use super::HasSideEffects;
7use crate::js_ast::*;
8use crate::util::is_js_identifier;
9use ast::*;
10
11/// Pushes a decl with a generated identifier to the hoisted result
12/// and returns an identifier expression for it.
13pub fn hoist_expr_if_needed(ctx: &mut BlockConvertContext, expr: Expr) -> Option<JsExpr> {
14 // if !expr.has_side_effects() {
15 // return Some(ctx.hoist_expression_contents(expr)?);
16 // }
17
18 let init = hoist_expr_contents(ctx, expr)?;
19
20 let id = ctx.generate_temp_id("_");
21 let range = init.range.clone();
22
23 ctx.add_statement(JsStatement::new(
24 JsStatementData::Declaration(JSDeclaration {
25 name: id.get(&ctx),
26 init: Some(init),
27 is_constant: true,
28 }),
29 range.clone(),
30 None,
31 ));
32
33 Some(JsExpr::new(
34 JsExprData::Identifier(id.get(&ctx)),
35 range,
36 None,
37 ))
38}
39
40/// Returning `None` from this means that the expression is noreturn, and that
41/// the rest of the statement never occurs and can be stripped.
42pub fn hoist_expr_contents(ctx: &mut BlockConvertContext, expr: Expr) -> Option<JsExpr> {
43 if expr.is_valid_js_expression() {
44 return Some(convert_valid_expr(ctx, expr));
45 }
46
47 match expr.data {
48 ExprData::Binary { op, left, right } => {
49 let left = hoist_expr_if_needed(ctx, *left)?;
50 let right = hoist_expr_if_needed(ctx, *right)?;
51
52 Some(JsExpr::new(
53 JsExprData::Binary {
54 op,
55 left: Box::new(left),
56 right: Box::new(right),
57 },
58 expr.range,
59 expr.comment,
60 ))
61 }
62
63 ExprData::Unary { op, value } => {
64 let value = hoist_expr_if_needed(ctx, *value)?;
65 Some(JsExpr::new(
66 JsExprData::Unary {
67 op,
68 value: Box::new(value),
69 },
70 expr.range,
71 expr.comment,
72 ))
73 }
74
75 ExprData::Field { base, field } => {
76 let base = hoist_expr_if_needed(ctx, *base)?;
77 let field = hoist_expr_if_needed(ctx, *field)?;
78
79 Some(JsExpr::new(
80 match field.data {
81 JsExprData::String(name) if is_js_identifier(name.as_str()) => {
82 JsExprData::Dot {
83 base: Box::new(base),
84 name,
85 }
86 }
87 _ => JsExprData::Index {
88 base: Box::new(base),
89 index: Box::new(field),
90 },
91 },
92 expr.range,
93 expr.comment,
94 ))
95 }
96
97 ExprData::Block(block) => hoist_block_to_expr(ctx, block, expr.range, expr.comment),
98
99 ExprData::If(_) => hoist_if_expr(ctx, expr),
100
101 ExprData::Return(maybe_expr) => {
102 let range = expr.range.clone();
103 let return_value = match maybe_expr {
104 Some(expr) => Some(hoist_expr_if_needed(ctx, *expr)?),
105 None => None,
106 };
107 ctx.add_statement(JsStatement::new(
108 JsStatementData::Return(return_value),
109 range,
110 None,
111 ));
112 None
113 }
114
115 ExprData::Call { base, arguments } => {
116 let base = hoist_expr_if_needed(ctx, *base)?;
117 let arguments = arguments
118 .into_iter()
119 .map(|x| hoist_expr_if_needed(ctx, x))
120 .collect::<Option<Vec<_>>>()?;
121
122 Some(JsExpr::new(
123 JsExprData::Call {
124 base: Box::new(base),
125 arguments,
126 },
127 expr.range,
128 expr.comment,
129 ))
130 }
131
132 ExprData::Fn { .. }
133 | ExprData::Builtin { .. }
134 | ExprData::Literal(_)
135 | ExprData::Identifier(_) => {
136 unreachable!("attempt to hoist an expression that does not require it")
137 }
138 }
139}
crates/js_ast/convert/hoist_if.rs deleted-113
......@@ -1,113 +0,0 @@
1use super::convert_valid_expr;
2use super::hoist_block_to_js_block;
3use super::hoist_expr_if_needed;
4use super::BlockConvertContext;
5use super::IsValidJSExpression;
6use super::TempId;
7use crate::js_ast::*;
8use ast::*;
9use log::Range;
10
11pub fn hoist_if_expr(ctx: &mut BlockConvertContext, expr: Expr) -> Option<JsExpr> {
12 let data = match expr.data {
13 ExprData::If(data) => data,
14 _ => unreachable!("hoist_if_expr called on non-if expression"),
15 };
16
17 let temporary = ctx.generate_temp_id("_if");
18
19 let decl = ctx.add_statement(JsStatement::new(
20 JsStatementData::Declaration(JSDeclaration {
21 name: temporary.get(&ctx),
22 init: None,
23 is_constant: false,
24 }),
25 expr.range.clone(),
26 None,
27 ));
28
29 let condition = hoist_expr_if_needed(ctx, *data.condition)?;
30 let mut did_assign = false;
31
32 let js_if = hoist_if_branch(
33 ctx,
34 data.yes,
35 data.no,
36 condition,
37 &mut did_assign,
38 &temporary,
39 );
40
41 ctx.add_statement(JsStatement::new(
42 JsStatementData::If(js_if),
43 expr.range.clone(),
44 None,
45 ));
46
47 if did_assign {
48 Some(JsExpr::new(
49 JsExprData::Identifier(temporary.get(&ctx)),
50 expr.range,
51 expr.comment,
52 ))
53 } else {
54 decl.delete(ctx);
55 Some(JsExpr::new(JsExprData::Null, expr.range, expr.comment))
56 }
57}
58
59pub fn hoist_if_branch(
60 ctx: &mut BlockConvertContext,
61 yes: BlockWithRange,
62 no: Option<BlockOrIf>,
63 condition: JsExpr,
64 did_assign: &mut bool,
65 temporary: &TempId,
66) -> JsIf {
67 let yes = hoist_block_to_js_block(ctx, yes.block, yes.range, &temporary, did_assign);
68
69 let no = match no {
70 Some(BlockOrIf::Block(no)) => Some(JsElse::Block(hoist_block_to_js_block(
71 ctx, no.block, no.range, &temporary, did_assign,
72 ))),
73 Some(BlockOrIf::If(else_if)) => {
74 // If the condition of this block has a side effect, we need to
75 // emit a block so that we can assign to the temporary variable.
76 //
77 // Otherwise, we can just emit the condition inline
78 if else_if.condition.is_valid_js_expression() {
79 let condition = convert_valid_expr(ctx, *else_if.condition.clone());
80 Some(JsElse::If(Box::new(hoist_if_branch(
81 ctx,
82 else_if.yes,
83 else_if.no,
84 condition,
85 did_assign,
86 temporary,
87 ))))
88 } else {
89 let mut temp_block = ctx.new_child();
90 let condition =
91 hoist_expr_if_needed(&mut temp_block, *else_if.condition.clone()).unwrap();
92
93 temp_block.add_statement(JsStatement::new(
94 JsStatementData::If(hoist_if_branch(
95 ctx,
96 else_if.yes,
97 else_if.no,
98 condition,
99 did_assign,
100 temporary,
101 )),
102 Range::EMPTY,
103 None,
104 ));
105
106 Some(JsElse::Block(temp_block.into()))
107 }
108 }
109 None => None,
110 };
111
112 JsIf { condition, yes, no }
113}
crates/js_ast/convert/is_valid_js_expression.rs deleted-91
......@@ -1,91 +0,0 @@
1use ast::*;
2
3/// "Valid" in this context means that the expression does not contain JS statements
4/// Meaning conversion is a direct map from a blight as node into a JS one
5///
6/// A valid expression should not go through any hoist logic, but if an expression
7/// isn't valid, then it means that it must be split into multiple JS statements
8/// inside of a block.
9pub trait IsValidJSExpression {
10 /// Can this be lowered to a JS ast without hoisting items
11 /// ex: a `return` expression is not a valid JS expression
12 fn is_valid_js_expression(&self) -> bool;
13}
14
15impl IsValidJSExpression for ExprData {
16 fn is_valid_js_expression(&self) -> bool {
17 match self {
18 // A block containing just expressions can be an expression
19 ExprData::Block(block) => block.is_valid_js_expression(),
20 ExprData::If(expr) => expr.is_valid_js_expression(),
21 ExprData::Return(_) => false,
22 ExprData::Builtin { id, .. } => match id {
23 BuiltinId::Raw => true,
24 },
25 ExprData::Binary { left, right, .. } => {
26 left.is_valid_js_expression() && right.is_valid_js_expression()
27 }
28 ExprData::Unary { value, .. } => value.is_valid_js_expression(),
29 ExprData::Fn { .. } => true,
30 ExprData::Call { base, arguments } => {
31 base.is_valid_js_expression()
32 && arguments
33 .iter()
34 .all(|argument| argument.is_valid_js_expression())
35 }
36 ExprData::Field { base, field } => {
37 base.is_valid_js_expression() && field.is_valid_js_expression()
38 }
39 ExprData::Literal(_) | ExprData::Identifier(_) => true,
40 }
41 }
42}
43
44impl IsValidJSExpression for Block {
45 fn is_valid_js_expression(&self) -> bool {
46 self.statements
47 .iter()
48 .all(|statement| statement.is_valid_js_expression())
49 }
50}
51
52impl IsValidJSExpression for Expr {
53 fn is_valid_js_expression(&self) -> bool {
54 self.data.is_valid_js_expression()
55 }
56}
57
58impl IsValidJSExpression for If {
59 fn is_valid_js_expression(&self) -> bool {
60 self.condition.is_valid_js_expression()
61 && self.yes.is_valid_js_expression()
62 && match &self.no {
63 Some(no) => no.is_valid_js_expression(),
64 None => true,
65 }
66 }
67}
68
69impl IsValidJSExpression for BlockOrIf {
70 fn is_valid_js_expression(&self) -> bool {
71 match self {
72 BlockOrIf::Block(block) => block.is_valid_js_expression(),
73 BlockOrIf::If(if_) => if_.is_valid_js_expression(),
74 }
75 }
76}
77
78impl IsValidJSExpression for BlockWithRange {
79 fn is_valid_js_expression(&self) -> bool {
80 self.block.is_valid_js_expression()
81 }
82}
83
84impl IsValidJSExpression for Statement {
85 fn is_valid_js_expression(&self) -> bool {
86 match self {
87 Statement::Expr(expr) => expr.data.is_valid_js_expression(),
88 _ => false,
89 }
90 }
91}
crates/js_ast/convert/mod.rs deleted-42
......@@ -1,42 +0,0 @@
1mod block_convert_context;
2mod convert_valid_block;
3mod convert_valid_expr;
4mod convert_valid_if;
5mod convert_valid_statement;
6mod hoist_block_to_expr;
7mod hoist_block_to_js_block;
8mod hoist_expr;
9mod hoist_if;
10mod is_valid_js_expression;
11mod process_function;
12mod process_statement;
13mod side_effects;
14mod tree_shake_statement_list;
15
16pub use block_convert_context::*;
17pub use convert_valid_block::*;
18pub use convert_valid_expr::*;
19pub use convert_valid_if::*;
20pub use convert_valid_statement::*;
21pub use hoist_block_to_expr::*;
22pub use hoist_block_to_js_block::*;
23pub use hoist_expr::*;
24pub use hoist_if::*;
25pub use is_valid_js_expression::*;
26pub use process_function::*;
27pub use process_statement::*;
28// pub use side_effects::*;
29pub use tree_shake_statement_list::*;
30
31use crate::js_ast::JsModule;
32use ast::Module;
33
34pub fn blight_to_js(blight: Module) -> JsModule {
35 let mut root_context = BlockConvertContext::new();
36
37 for statement in blight.statements {
38 process_statement(&mut root_context, statement).unwrap();
39 }
40
41 root_context.into()
42}
crates/js_ast/convert/process_function.rs deleted-65
......@@ -1,65 +0,0 @@
1use super::convert_valid_expr;
2use super::hoist_expr_if_needed;
3use super::process_statement;
4use super::BlockConvertContext;
5use super::IsValidJSExpression;
6use crate::js_ast::*;
7use ast::*;
8use log::Range;
9
10pub fn process_function_to_arrow(
11 ctx: &mut BlockConvertContext,
12 body: Expr,
13 params: Vec<Param>,
14 range: Range,
15 comment: Option<Comment>,
16) -> JsExpr {
17 // A block that is a valid expression will just become
18 // that expression. We can convert multiple expressions
19 // into a comma expression.
20 let body = if body.is_valid_js_expression() {
21 JsArrowBody::Expr(convert_valid_expr(ctx, body))
22 } else {
23 let mut child = ctx.new_child();
24
25 match body.data {
26 ExprData::Block(block) => {
27 for statement in block.statements {
28 if let None = process_statement(&mut child, statement) {
29 break;
30 };
31 }
32
33 if block.has_implicit_return {
34 if let Some(expr) = child.take_last_statement_if_expr() {
35 child.add_statement(JsStatement::new(
36 JsStatementData::Return(Some(expr)),
37 range.clone(),
38 None,
39 ));
40 }
41 }
42 }
43 _ => {
44 if let Some(expr) = hoist_expr_if_needed(&mut child, body) {
45 child.add_statement(JsStatement::new(
46 JsStatementData::Return(Some(expr)),
47 range.clone(),
48 None,
49 ));
50 }
51 }
52 }
53
54 JsArrowBody::Block(child.into())
55 };
56
57 JsExpr::new(
58 JsExprData::ArrowFunction {
59 params: params.into_iter().map(|x| x.name).collect(),
60 body: Box::new(body),
61 },
62 range.clone(),
63 comment.map(|x| x.into()),
64 )
65}
crates/js_ast/convert/process_statement.rs deleted-98
......@@ -1,98 +0,0 @@
1use super::convert_valid_statement;
2use super::hoist_expr_contents;
3use super::process_function_to_arrow;
4use super::BlockConvertContext;
5use super::IsValidJSExpression;
6use crate::js_ast::*;
7use ast::*;
8
9pub fn process_statement(ctx: &mut BlockConvertContext, statement: Statement) -> Option<()> {
10 if statement.is_valid_js_expression() {
11 let result = convert_valid_statement(ctx, statement);
12 ctx.add_statement(result);
13 return Some(());
14 }
15
16 match statement {
17 Statement::VarDecl(decl) => {
18 // visit init before name, this way allows for:
19 //
20 // let a = a;
21 //
22 // and `a` will be looked up before it is re-declared.
23 // we will also return in the noreturn case
24 let init = hoist_expr_contents(ctx, decl.init)?;
25 let name = ctx.declare_identifier(decl.name);
26
27 let decl_data = JSDeclaration {
28 name: name.get(&ctx),
29 init: Some(init),
30 is_constant: decl.is_constant,
31 };
32 let stmt = JsStatement::new(
33 if decl.is_pub {
34 JsStatementData::Export(JsExport::Declaration(Box::new(decl_data)))
35 } else {
36 JsStatementData::Declaration(decl_data)
37 },
38 decl.range,
39 decl.comment.map(|x| x.into()),
40 );
41
42 ctx.add_statement(stmt);
43
44 Some(())
45 }
46 Statement::Expr(expr) => {
47 if let Some(expr) = hoist_expr_contents(ctx, expr) {
48 let range = expr.range.clone();
49 ctx.add_statement(JsStatement::new(
50 JsStatementData::Expression(expr),
51 range,
52 None,
53 ));
54 Some(())
55 } else {
56 None
57 }
58 }
59 Statement::FnDecl(FnDecl {
60 name,
61 params,
62 body,
63 range,
64 comment,
65 is_pub,
66 ..
67 }) => {
68 let name = ctx.declare_identifier(name);
69
70 let processed = Some(process_function_to_arrow(
71 ctx,
72 body,
73 params,
74 range.clone(),
75 None,
76 ));
77
78 let decl_data = JSDeclaration {
79 name: name.get(&ctx),
80 init: processed,
81 is_constant: true,
82 };
83
84 ctx.add_statement(JsStatement::new(
85 if is_pub {
86 JsStatementData::Export(JsExport::Declaration(Box::new(decl_data)))
87 } else {
88 JsStatementData::Declaration(decl_data)
89 },
90 range,
91 comment.map(|x| x.into()),
92 ));
93
94 Some(())
95 }
96 Statement::Comment(_) => Some(()),
97 }
98}
crates/js_ast/convert/side_effects.rs deleted-91
......@@ -1,91 +0,0 @@
1use ast::*;
2
3pub trait HasSideEffects {
4 /// Can this item be removed from the AST without changing the program's behavior?
5 fn has_side_effects(&self) -> bool;
6}
7
8impl HasSideEffects for ExprData {
9 fn has_side_effects(&self) -> bool {
10 match self {
11 ExprData::Literal(_) => false,
12 ExprData::Identifier(_) => false,
13
14 ExprData::Block(block) => block.has_side_effects(),
15 ExprData::Binary { left, right, .. } => {
16 left.has_side_effects() || right.has_side_effects()
17 }
18 ExprData::Unary { value, .. } => value.has_side_effects(),
19 ExprData::If(ifexpr) => ifexpr.has_side_effects(),
20
21 // note: a function *declaration* has no side effects, but the call *can*
22 ExprData::Fn { .. } => false,
23
24 // TODO(@paperdave): implement detection of pure functions
25 ExprData::Call { base, arguments } => {
26 base.has_side_effects() || arguments.iter().any(|x| x.has_side_effects())
27 }
28
29 // note: we assume field access has side effects, ie: a getter doesn't throw
30 ExprData::Field { base, field } => base.has_side_effects() || field.has_side_effects(),
31
32 // note: we assume the code in @raw always has side effects
33 ExprData::Builtin { id, .. } => match id {
34 BuiltinId::Raw => true,
35 },
36
37 ExprData::Return(_) => true,
38 }
39 }
40}
41
42impl HasSideEffects for Statement {
43 fn has_side_effects(&self) -> bool {
44 match self {
45 Statement::Expr(expr) => expr.has_side_effects(),
46 Statement::VarDecl(decl) => decl.init.has_side_effects(),
47 Statement::Comment(_) => false,
48 Statement::FnDecl(_) => false,
49 }
50 }
51}
52
53impl HasSideEffects for Block {
54 fn has_side_effects(&self) -> bool {
55 self.statements
56 .iter()
57 .any(|statement| statement.has_side_effects())
58 }
59}
60
61impl HasSideEffects for Expr {
62 fn has_side_effects(&self) -> bool {
63 self.data.has_side_effects()
64 }
65}
66
67impl HasSideEffects for If {
68 fn has_side_effects(&self) -> bool {
69 self.condition.has_side_effects()
70 || self.yes.has_side_effects()
71 || match &self.no {
72 Some(no) => no.has_side_effects(),
73 None => false,
74 }
75 }
76}
77
78impl HasSideEffects for BlockOrIf {
79 fn has_side_effects(&self) -> bool {
80 match self {
81 BlockOrIf::Block(block) => block.has_side_effects(),
82 BlockOrIf::If(if_) => if_.has_side_effects(),
83 }
84 }
85}
86
87impl HasSideEffects for BlockWithRange {
88 fn has_side_effects(&self) -> bool {
89 self.block.has_side_effects()
90 }
91}
crates/js_ast/convert/tree_shake_statement_list.rs deleted-31
......@@ -1,31 +0,0 @@
1use crate::js_ast::*;
2
3/// Removes statements that are not needed from a JS block. Very limited.
4///
5/// BlockConvertContext will automatically execute this when converting
6/// itself into a Statement list or other data structure.
7///
8/// What this does:
9/// - Remove discarded literals `null` -> ``
10pub fn tree_shake_statement_list(list: &mut Vec<JsStatement>) {
11 let mut set_i = 0;
12 for i in 0..list.len() {
13 let stmt = list.get(i).unwrap();
14 let should_keep = match &stmt.data {
15 JsStatementData::Expression(expr) => !expr.is_literal(),
16 JsStatementData::Return(_) => true,
17 JsStatementData::Export(_) => true,
18 JsStatementData::Declaration(_) => true,
19 JsStatementData::Function(_) => true,
20 JsStatementData::If(_) => true,
21 };
22
23 if should_keep {
24 if set_i != i {
25 list.swap(i, set_i)
26 }
27 set_i += 1;
28 }
29 }
30 list.truncate(set_i)
31}
crates/js_ast/js_ast.rs deleted-198
......@@ -1,198 +0,0 @@
1use ast::{BinaryOperator, Comment, UnaryOperator};
2use log::Range;
3use num::bigint::BigInt;
4
5#[derive(Debug, PartialEq, Clone)]
6pub struct JsExpr {
7 pub data: JsExprData,
8 pub range: Range,
9 pub comment: Option<Comment>,
10}
11
12#[derive(Debug, PartialEq, Clone)]
13pub struct JsKV {
14 pub key: JsExpr,
15 pub value: JsExpr,
16}
17
18#[derive(Debug, PartialEq, Clone)]
19pub enum JsExprData {
20 /// node emitted by the '@raw' building
21 Raw(String),
22 // Litearls
23 Number(f64),
24 BigInt(BigInt),
25 String(String),
26 Bool(bool),
27 Array(Vec<JsExpr>),
28 Object(Vec<JsKV>),
29 Null,
30 Undefined,
31
32 Identifier(String),
33 Unary {
34 op: UnaryOperator,
35 value: Box<JsExpr>,
36 },
37 Binary {
38 op: BinaryOperator,
39 left: Box<JsExpr>,
40 right: Box<JsExpr>,
41 },
42 Dot {
43 base: Box<JsExpr>,
44 name: String,
45 },
46 Index {
47 base: Box<JsExpr>,
48 index: Box<JsExpr>,
49 },
50 Call {
51 base: Box<JsExpr>,
52 arguments: Vec<JsExpr>,
53 },
54 Function {
55 name: Option<String>,
56 params: Vec<String>,
57 body: Box<JsExpr>,
58 },
59 ArrowFunction {
60 params: Vec<String>,
61 body: Box<JsArrowBody>,
62 },
63 Ternary {
64 condition: Box<JsExpr>,
65 yes: Box<JsExpr>,
66 no: Option<Box<JsExpr>>,
67 },
68 Comma {
69 items: Vec<JsExpr>,
70 },
71 Void(Box<JsExpr>),
72}
73
74#[derive(Debug, PartialEq, Clone)]
75pub struct JsStatement {
76 pub data: JsStatementData,
77 pub range: Range,
78 pub comment: Option<Comment>,
79}
80
81#[derive(Debug, PartialEq, Clone)]
82pub struct JSFunctionDeclaration {
83 pub name: String,
84 pub params: Vec<String>,
85 pub block: JsBlock,
86}
87
88#[derive(Debug, PartialEq, Clone)]
89pub struct JSDeclaration {
90 pub name: String,
91 pub init: Option<JsExpr>,
92 pub is_constant: bool,
93}
94
95#[derive(Debug, PartialEq, Clone)]
96pub struct JsIf {
97 pub condition: JsExpr,
98 pub yes: JsBlock,
99 pub no: Option<JsElse>,
100}
101
102#[derive(Debug, PartialEq, Clone)]
103pub enum JsElse {
104 Expr(JsExpr),
105 Block(JsBlock),
106 If(Box<JsIf>),
107}
108
109#[derive(Debug, PartialEq, Clone)]
110pub enum JsStatementData {
111 Expression(JsExpr),
112 Return(Option<JsExpr>),
113 Export(JsExport),
114 Declaration(JSDeclaration),
115 Function(JSFunctionDeclaration),
116 If(JsIf),
117}
118
119#[derive(Debug, PartialEq, Clone)]
120pub enum JsExport {
121 Named(Vec<JSNamedExport>),
122 Function(Box<JSFunctionDeclaration>),
123 Declaration(Box<JSDeclaration>),
124 Default(JsExpr),
125}
126
127#[derive(Debug, PartialEq, Clone)]
128pub struct JSNamedExport {
129 pub identifier: String,
130 pub export_name: String,
131}
132
133#[derive(Debug, PartialEq, Clone)]
134pub struct JsBlock {
135 pub statements: Vec<JsStatement>,
136 pub range: Range,
137}
138
139impl JsExpr {
140 pub fn new(data: JsExprData, range: Range, comment: Option<Comment>) -> Self {
141 Self {
142 data,
143 range,
144 comment,
145 }
146 }
147
148 pub fn is_literal(&self) -> bool {
149 match &self.data {
150 JsExprData::Number(_) => true,
151 JsExprData::BigInt(_) => true,
152 JsExprData::String(_) => true,
153 JsExprData::Bool(_) => true,
154 JsExprData::Null => true,
155 JsExprData::Undefined => true,
156 _ => false,
157 }
158 }
159}
160
161impl JsExprData {
162 /// Helper for testing
163 pub fn str(value: &str) -> Self {
164 Self::String(value.to_string())
165 }
166
167 /// Helper for testing
168 pub fn str_id(value: &str) -> Self {
169 Self::Identifier(value.to_string())
170 }
171}
172
173impl JsBlock {
174 pub fn new(statements: Vec<JsStatement>, range: Range) -> Self {
175 Self { statements, range }
176 }
177}
178
179impl JsStatement {
180 pub fn new(data: JsStatementData, range: Range, comment: Option<Comment>) -> Self {
181 Self {
182 data,
183 range,
184 comment,
185 }
186 }
187}
188
189#[derive(Debug, PartialEq, Clone)]
190pub enum JsArrowBody {
191 Block(JsBlock),
192 Expr(JsExpr),
193}
194
195#[derive(Debug, PartialEq, Clone)]
196pub struct JsModule {
197 pub statements: Vec<JsStatement>,
198}
crates/js_ast/lib.rs deleted-7
......@@ -1,7 +0,0 @@
1mod convert;
2mod js_ast;
3mod util;
4
5pub use convert::*;
6pub use js_ast::*;
7pub use util::*;
crates/js_ast/util.rs deleted-17
......@@ -1,17 +0,0 @@
1/// TODO(@paperdave): handle the rest of the identifiers
2pub fn is_js_identifier(s: &str) -> bool {
3 if s.is_empty() {
4 return false;
5 }
6 let mut chars = s.chars();
7 let first = chars.next().unwrap();
8 if !first.is_ascii_alphabetic() && first != '_' {
9 return false;
10 }
11 for c in chars {
12 if !c.is_ascii_alphanumeric() && c != '_' {
13 return false;
14 }
15 }
16 true
17}
crates/js_printer/Cargo.toml deleted-14
......@@ -1,14 +0,0 @@
1[package]
2name = "js_printer"
3version = "0.1.0"
4edition = "2021"
5
6[lib]
7path = "./lib.rs"
8
9[dependencies]
10miette.workspace = true
11thiserror.workspace = true
12num.workspace = true
13
14js_ast = { path = "../js_ast" }
crates/js_printer/lib.rs deleted-377
......@@ -1,377 +0,0 @@
1mod printer_tests;
2
3use js_ast::*;
4use std::io::{Result, Write};
5
6pub trait Print {
7 fn print(&self, writer: &mut dyn Write) -> Result<()>;
8
9 fn print_to_string(&self) -> String {
10 let mut buffer = Vec::new();
11 self.print(&mut buffer).unwrap();
12 String::from_utf8(buffer).unwrap()
13 }
14}
15
16// the printer:
17
18fn print_module_clause_identifier(writer: &mut dyn Write, export_name: &str) -> Result<()> {
19 // TODO(@paperdave): Escape invalid identifiers
20 if js_ast::is_js_identifier(export_name) {
21 write!(writer, "{}", export_name)
22 } else {
23 todo!("call JSON.stringify on identifier")
24 }
25}
26
27fn print_js_string(writer: &mut dyn Write, string: &str) -> Result<()> {
28 write!(writer, "\"")?;
29 // TODO(@paperdave): Instead of writing each character, this should write
30 // chunks between the escapes instead of calling write! on each character individually.
31 for c in string.chars() {
32 match c {
33 '\n' => write!(writer, "\\n")?,
34 '\r' => write!(writer, "\\r")?,
35 '\t' => write!(writer, "\\t")?,
36 '"' => write!(writer, "\\\"")?,
37 '\\' => write!(writer, "\\\\")?,
38 '\u{0008}' => write!(writer, "\\b")?,
39 '\u{000C}' => write!(writer, "\\f")?,
40 '\u{0000}'..='\u{001F}' => write!(writer, "\\u{:04x}", c as u32)?,
41 // TODO(@paperdave): I think there are some larger code points that need to be escaped?
42 _ => write!(writer, "{}", c)?,
43 }
44 }
45 write!(writer, "\"")
46}
47
48impl Print for JsStatement {
49 fn print(&self, w: &mut dyn Write) -> Result<()> {
50 match &self.data {
51 js_ast::JsStatementData::Expression(expr) => {
52 expr.print(w)?;
53 write!(w, ";")
54 }
55 js_ast::JsStatementData::Return(expr) => {
56 write!(w, "return")?;
57 if let Some(expr) = expr {
58 write!(w, " ")?;
59 expr.print(w)?;
60 }
61 write!(w, ";")?;
62 Ok(())
63 }
64 js_ast::JsStatementData::Export(export) => export.print(w),
65 js_ast::JsStatementData::Declaration(decl) => decl.print(w),
66 js_ast::JsStatementData::Function(function) => function.print(w),
67 js_ast::JsStatementData::If(js_if) => js_if.print(w),
68 }
69 }
70}
71
72impl Print for JsExport {
73 fn print(&self, w: &mut dyn Write) -> Result<()> {
74 match self {
75 JsExport::Named(exports) => {
76 write!(w, "export{{")?;
77 for (i, export) in exports.iter().enumerate() {
78 if i > 0 {
79 write!(w, ",")?;
80 }
81 export.print(w)?;
82 }
83 Ok(())
84 }
85 JsExport::Function(function) => {
86 write!(w, "export ")?;
87 function.print(w)
88 }
89 JsExport::Declaration(declaration) => {
90 write!(w, "export ")?;
91 declaration.print(w)
92 }
93 JsExport::Default(expr) => {
94 write!(w, "export default ")?;
95 expr.print(w)
96 }
97 }
98 }
99}
100
101impl Print for JSNamedExport {
102 fn print(&self, writer: &mut dyn Write) -> Result<()> {
103 if self.export_name == self.identifier {
104 write!(writer, "{}", self.identifier)
105 } else {
106 write!(writer, "{} as ", self.identifier)?;
107 print_module_clause_identifier(writer, &self.export_name)
108 }
109 }
110}
111
112impl Print for JSDeclaration {
113 fn print(&self, writer: &mut dyn Write) -> Result<()> {
114 write!(
115 writer,
116 "{} {}",
117 if self.is_constant { "const" } else { "let" },
118 self.name,
119 )?;
120 if let Some(init) = &self.init {
121 write!(writer, " = ")?;
122 init.print(writer)?;
123 }
124 write!(writer, ";")?;
125 Ok(())
126 }
127}
128
129impl Print for JsExpr {
130 fn print(&self, writer: &mut dyn Write) -> Result<()> {
131 self.data.print(writer)
132 }
133}
134
135impl Print for JsExprData {
136 fn print(&self, writer: &mut dyn Write) -> Result<()> {
137 match &self {
138 JsExprData::Raw(data) => write!(writer, "{}", data),
139 JsExprData::Number(number) => write!(writer, "{}", number),
140 JsExprData::BigInt(bigint) => write!(writer, "{}", bigint),
141 JsExprData::String(string) => print_js_string(writer, string.as_str()),
142 JsExprData::Bool(bool) => write!(writer, "{}", bool),
143 JsExprData::Null => write!(writer, "null"),
144 JsExprData::Undefined => write!(writer, "undefined"),
145 JsExprData::Identifier(str) => write!(writer, "{}", str),
146 JsExprData::Array(items) => {
147 write!(writer, "[")?;
148 for (i, item) in items.iter().enumerate() {
149 if i > 0 {
150 write!(writer, ",")?;
151 }
152 item.print(writer)?;
153 }
154 write!(writer, "]")
155 }
156 JsExprData::Object(items) => {
157 write!(writer, "{{")?;
158 for (i, kv) in items.iter().enumerate() {
159 if i > 0 {
160 write!(writer, ",")?;
161 }
162
163 kv.print(writer)?;
164 }
165 write!(writer, "}}")
166 }
167 JsExprData::Unary { op, value } => {
168 write!(writer, "{}", op.to_str())?;
169 value.print(writer)
170 }
171 JsExprData::Binary { op, left, right } => {
172 // TODO: figure out when we dont need to wrap
173 let wrap = true;
174 if wrap {
175 write!(writer, "(")?;
176 }
177 left.print(writer)?;
178 write!(writer, "{}", op.to_str())?;
179 right.print(writer)?;
180 if wrap {
181 write!(writer, ")")?;
182 }
183 Ok(())
184 }
185 JsExprData::Dot { base, name } => {
186 base.print(writer)?;
187 write!(writer, ".{}", name)
188 }
189 JsExprData::Index { base, index } => {
190 base.print(writer)?;
191 write!(writer, "[")?;
192 index.print(writer)?;
193 write!(writer, "]")
194 }
195 JsExprData::Call { base, arguments } => {
196 base.print(writer)?;
197 write!(writer, "(")?;
198 for (i, argument) in arguments.iter().enumerate() {
199 if i > 0 {
200 write!(writer, ",")?;
201 }
202 argument.print(writer)?;
203 }
204 write!(writer, ")")
205 }
206 JsExprData::Function { body, name, params } => {
207 write!(writer, "function ")?;
208 if let Some(name) = name {
209 write!(writer, "{}", name)?;
210 }
211 write!(writer, "(")?;
212 for (i, param) in params.iter().enumerate() {
213 if i > 0 {
214 write!(writer, ",")?;
215 }
216 write!(writer, "{}", param)?;
217 }
218 write!(writer, ")")?;
219 body.print(writer)
220 }
221 JsExprData::ArrowFunction { body, params } => {
222 let print_parens = params.len() != 1;
223 if print_parens {
224 write!(writer, "(")?;
225 }
226 for (i, param) in params.iter().enumerate() {
227 if i > 0 {
228 write!(writer, ",")?;
229 }
230 write!(writer, "{}", param)?;
231 }
232 if print_parens {
233 write!(writer, ")")?;
234 }
235 write!(writer, "=>")?;
236 body.print(writer)
237 }
238 JsExprData::Ternary { condition, yes, no } => {
239 let wrap = false;
240 if wrap {
241 write!(writer, "(")?;
242 }
243 condition.print(writer)?;
244 write!(writer, "?")?;
245 yes.print(writer)?;
246 write!(writer, ":")?;
247 if let Some(no) = no {
248 no.print(writer)?;
249 } else {
250 write!(writer, "null")?;
251 }
252 if wrap {
253 write!(writer, ")")?;
254 }
255 Ok(())
256 }
257 JsExprData::Comma { items } => {
258 let wrap = true;
259 if wrap {
260 write!(writer, "(")?;
261 }
262 for (i, item) in items.iter().enumerate() {
263 if i > 0 {
264 write!(writer, ",")?;
265 }
266 item.print(writer)?;
267 }
268 if wrap {
269 write!(writer, ")")?;
270 }
271 Ok(())
272 }
273 JsExprData::Void(expr) => {
274 write!(writer, "void ")?;
275 expr.print(writer)
276 }
277 }
278 }
279}
280
281impl Print for JSFunctionDeclaration {
282 fn print(&self, writer: &mut dyn Write) -> Result<()> {
283 write!(writer, "function {}", self.name)?;
284 write!(writer, "(")?;
285 for (i, param) in self.params.iter().enumerate() {
286 if i > 0 {
287 write!(writer, ",")?;
288 }
289 write!(writer, "{}", param)?;
290 }
291 write!(writer, ")")?;
292 self.block.print(writer)
293 }
294}
295
296impl Print for JsBlock {
297 fn print(&self, writer: &mut dyn Write) -> Result<()> {
298 write!(writer, "{{")?;
299 for statement in self.statements.iter() {
300 statement.print(writer)?;
301 }
302 write!(writer, "}}")
303 }
304}
305
306impl Print for JsIf {
307 fn print(&self, writer: &mut dyn Write) -> Result<()> {
308 write!(writer, "if(")?;
309 self.condition.print(writer)?;
310 write!(writer, ")")?;
311 self.yes.print(writer)?;
312 if let Some(no) = &self.no {
313 no.print(writer)?;
314 }
315 Ok(())
316 }
317}
318
319impl Print for JsElse {
320 fn print(&self, writer: &mut dyn Write) -> Result<()> {
321 match self {
322 JsElse::If(js_if) => {
323 write!(writer, "else ")?;
324 js_if.print(writer)
325 }
326 JsElse::Block(block) => {
327 write!(writer, "else")?;
328 block.print(writer)
329 }
330 JsElse::Expr(expr) => {
331 write!(writer, "else ")?;
332 expr.print(writer)
333 }
334 }
335 }
336}
337
338impl Print for JsKV {
339 fn print(&self, writer: &mut dyn Write) -> Result<()> {
340 match &self.key.data {
341 JsExprData::String(string) => {
342 if is_js_identifier(string) {
343 write!(writer, "{}", string)?;
344 } else {
345 print_js_string(writer, string.as_str())?;
346 }
347 }
348 _ => {
349 write!(writer, "[")?;
350 self.key.print(writer)?;
351 write!(writer, "]")?;
352 }
353 }
354 write!(writer, ":")?;
355 self.key.print(writer)?;
356 write!(writer, ":")?;
357 self.value.print(writer)
358 }
359}
360
361impl Print for JsModule {
362 fn print(&self, writer: &mut dyn Write) -> Result<()> {
363 for statement in self.statements.iter() {
364 statement.print(writer)?;
365 }
366 Ok(())
367 }
368}
369
370impl Print for JsArrowBody {
371 fn print(&self, writer: &mut dyn Write) -> Result<()> {
372 match self {
373 JsArrowBody::Block(block) => block.print(writer),
374 JsArrowBody::Expr(expr) => expr.print(writer),
375 }
376 }
377}
crates/js_printer/printer_tests.rs deleted-53
......@@ -1,53 +0,0 @@
1#![cfg(test)]
2use crate::Print;
3
4use js_ast::JsExprData as E;
5use num::{bigint::Sign, BigInt};
6
7macro_rules! assert_printed {
8 ($value:expr, $expected:expr) => {
9 assert_eq!($value.print_to_string(), $expected);
10 };
11}
12
13#[test]
14fn number() {
15 assert_printed!(E::Number(1.0), "1");
16 assert_printed!(E::Number(1.5), "1.5");
17 assert_printed!(E::Number(1.5e10), "15000000000");
18 assert_printed!(E::Number(1.5e-10), "0.00000000015");
19 assert_printed!(E::Number(-1.0), "-1");
20 assert_printed!(E::Number(-1.5), "-1.5");
21 assert_printed!(E::Number(0f64), "0");
22}
23
24#[test]
25fn bigint() {
26 assert_printed!(E::BigInt(BigInt::new(Sign::Plus, vec![1])), "1n");
27 assert_printed!(
28 E::BigInt(BigInt::new(Sign::Plus, vec![10000000])),
29 "10000000n"
30 );
31 assert_printed!(
32 E::BigInt(BigInt::new(Sign::Minus, vec![10000000])),
33 "-10000000n"
34 );
35}
36
37#[test]
38fn boolean() {
39 assert_printed!(E::Bool(true), "true");
40 assert_printed!(E::Bool(false), "false");
41}
42
43#[test]
44fn string() {
45 assert_printed!(E::str(""), "\"\"");
46 assert_printed!(E::str("hello"), "\"hello\"");
47 assert_printed!(E::str("hello\nworld"), "\"hello\\nworld\"");
48}
49
50#[test]
51fn identifier() {
52 assert_printed!(E::str_id("hello"), "hello");
53}
crates/log/Cargo.toml deleted-11
......@@ -1,11 +0,0 @@
1[package]
2name = "log"
3edition = "2021"
4
5[lib]
6path="./log.rs"
7
8[dependencies]
9miette.workspace = true
10thiserror.workspace = true
11url.workspace = true
crates/log/log.rs deleted-88
......@@ -1,88 +0,0 @@
1use std::{fs, path::Path, rc::Rc};
2
3use miette::{IntoDiagnostic, Result};
4use url::Url;
5
6#[derive(Debug, PartialEq, Clone, Copy)]
7pub struct Range {
8 pub start: usize,
9 pub length: usize,
10}
11
12impl Range {
13 /// TODO(@paperdave): Remove this. every usage of this is incorrect.
14 pub const EMPTY: Range = Range {
15 start: 0,
16 length: 0,
17 };
18
19 pub fn new(start: usize, length: usize) -> Self {
20 Self { start, length }
21 }
22
23 pub fn len(&self) -> usize {
24 self.length
25 }
26}
27
28// TODO(@lino-levan): make this loggable nicely
29#[derive(Debug, Clone)]
30pub struct Message<'a> {
31 pub source: &'a Source,
32 pub range: Range,
33 pub message: String,
34 pub notes: Vec<Note<'a>>,
35}
36
37#[derive(Debug, Clone)]
38pub struct Note<'a> {
39 pub location: Option<RangeAndSource<'a>>,
40 pub message: String,
41}
42
43#[derive(Debug, Clone)]
44pub struct RangeAndSource<'a> {
45 pub range: Range,
46 pub source: &'a Source,
47}
48
49#[derive(Debug)]
50pub struct Source {
51 pub url: Url,
52 pub code: String,
53}
54
55impl Source {
56 pub fn new(url: Url, source_code: String) -> Rc<Self> {
57 Rc::new(Source {
58 url,
59 code: source_code,
60 })
61 }
62
63 pub fn from_file(path: &Path) -> Result<Rc<Self>> {
64 let source_code = fs::read_to_string(path).into_diagnostic()?;
65 let url = Url::from_file_path(path).unwrap();
66 Ok(Rc::new(Source {
67 code: source_code,
68 url,
69 }))
70 }
71
72 pub fn from_url(url: Url) -> Result<Rc<Self>> {
73 match url.scheme() {
74 "file" => {
75 // TODO: how to return this error, into_diagnostic() doesn't exist
76 let file_path = url.to_file_path().unwrap();
77 let source_code = fs::read_to_string(file_path).into_diagnostic()?;
78 Ok(Rc::new(Source {
79 code: source_code,
80 url,
81 }))
82 }
83 _ => {
84 todo!("Fetch from URL: {}", url)
85 }
86 }
87 }
88}
crates/parser/Cargo.toml deleted-16
......@@ -1,16 +0,0 @@
1[package]
2name = "parser"
3version = "0.1.0"
4edition = "2021"
5
6[lib]
7path = "./parser.rs"
8
9[dependencies]
10miette.workspace = true
11thiserror.workspace = true
12num.workspace = true
13
14ast = { path = "../ast" }
15tokenizer = { path = "../tokenizer" }
16log = { path = "../log" }
crates/parser/parse_block.rs deleted-84
......@@ -1,84 +0,0 @@
1use crate::{parse_statement::parse_statement, Parser};
2use ast::Block;
3use miette::Result;
4use tokenizer::TokenType;
5
6/// Precondition: The first token exists and is '{'
7pub fn parse_block(p: &mut Parser) -> Result<Block> {
8 match p.next() {
9 Some(TokenType::OpenBrace) => (),
10 _ => unreachable!(),
11 }
12
13 let mut statements = Vec::new();
14
15 let was_top_level = p.is_top_level;
16 p.is_top_level = false;
17
18 'outer: loop {
19 let is_statement_like_expression = p
20 .peek()
21 .map_or(false, |x| x.is_start_of_statement_like_expression());
22
23 match p.peek() {
24 Some(TokenType::ClosedBrace) => {
25 p.next();
26 p.is_top_level = was_top_level;
27 return Ok(Block {
28 statements,
29 has_implicit_return: false,
30 });
31 }
32 _ => (),
33 }
34
35 let statement = match parse_statement(p) {
36 Ok(statement) => statement,
37 Err(err) => {
38 p.is_top_level = was_top_level;
39 return Err(err);
40 }
41 };
42 statements.push(statement.clone());
43
44 match p.peek() {
45 Some(TokenType::Semicolon) => loop {
46 p.next();
47 match p.peek() {
48 Some(TokenType::ClosedBrace) => {
49 p.next();
50 p.is_top_level = was_top_level;
51 return Ok(Block {
52 statements,
53 has_implicit_return: false,
54 });
55 }
56 _ => continue 'outer,
57 }
58 },
59 Some(TokenType::ClosedBrace) => {
60 p.next();
61 p.is_top_level = was_top_level;
62
63 if !statement.is_expression() {
64 return Err(p
65 .error("Expected semicolon or '}' after statement".to_string())
66 .into());
67 }
68
69 return Ok(Block {
70 statements,
71 has_implicit_return: true,
72 });
73 }
74 _ => {
75 if statement.require_semicolon(is_statement_like_expression) {
76 p.is_top_level = was_top_level;
77 return Err(p
78 .error("Expected semicolon or '}' after statement".to_string())
79 .into());
80 }
81 }
82 };
83 }
84}
crates/parser/parse_call_arguments.rs deleted-36
......@@ -1,36 +0,0 @@
1use ast::Expr;
2use miette::Result;
3use tokenizer::TokenType;
4
5use crate::{parse_expr::parse_expr, Parser};
6
7/// Precondition: The '(' was already consumed
8pub fn parse_call_arguments(p: &mut Parser) -> Result<Vec<Expr>> {
9 let mut args = Vec::new();
10
11 if let Some(TokenType::ClosedParen) = p.peek() {
12 _ = p.next();
13 return Ok(args);
14 }
15
16 loop {
17 let arg = parse_expr(p)?;
18 args.push(arg);
19
20 match p.peek() {
21 Some(TokenType::Comma) => {
22 _ = p.next();
23 continue;
24 }
25 Some(TokenType::ClosedParen) => {
26 _ = p.next();
27 return Ok(args);
28 }
29 _ => {
30 return Err(p
31 .error("Expected ',' or ')' after argument".to_string())
32 .into())
33 }
34 }
35 }
36}
crates/parser/parse_declaration.rs deleted-51
......@@ -1,51 +0,0 @@
1use ast::{DocComment, Type, VarDecl};
2use log::Range;
3use miette::Result;
4use tokenizer::TokenType;
5
6use crate::parse_expr::parse_expr;
7use crate::parse_type::parse_type;
8use crate::Parser;
9
10/// Precondition: The first token exists and is either 'var' or 'let'
11pub fn parse_declaration(
12 p: &mut Parser,
13 is_pub: bool,
14 comment: Option<DocComment>,
15) -> Result<VarDecl> {
16 let start = p.index;
17 let is_constant = match p.next() {
18 Some(TokenType::Var) => false,
19 Some(TokenType::Let) => true,
20 _ => unreachable!(),
21 };
22 let name = match p.next() {
23 Some(TokenType::Identifier(name)) => name,
24 Some(tok) => return Err(p.error(format!("Unexpected {}", tok)).into()),
25 None => return Err(p.error(format!("Unexpected end of file")).into()),
26 };
27 let return_type = match p.peek() {
28 Some(TokenType::Colon) => {
29 p.next();
30 parse_type(p)?
31 }
32 _ => Type::Infer,
33 };
34 let init = match p.next() {
35 Some(TokenType::Equal) => parse_expr(p)?,
36 _ => {
37 return Err(p
38 .error("Variable must have an initializer".to_string())
39 .into())
40 }
41 };
42 Ok(VarDecl {
43 name: name.clone(),
44 data_type: return_type,
45 init,
46 is_constant,
47 is_pub,
48 comment,
49 range: Range::new(start, p.index - start),
50 })
51}
crates/parser/parse_doc_comment.rs deleted-19
......@@ -1,19 +0,0 @@
1use ast::DocComment;
2use log::Range;
3use tokenizer::TokenType;
4
5use crate::Parser;
6
7pub fn maybe_read_doc_comment(p: &mut Parser) -> Option<DocComment> {
8 match p.peek() {
9 Some(TokenType::DocComment(data)) => {
10 let start = p.index;
11 p.next();
12 Some(DocComment {
13 data,
14 range: Range::new(start, p.index - start),
15 })
16 }
17 _ => None,
18 }
19}
crates/parser/parse_expr.rs deleted-71
......@@ -1,71 +0,0 @@
1use ast::{Associativity, BinaryOperator, Expr, ExprData, OperatorLevel};
2use log::Range;
3use miette::Result;
4
5use crate::parse_expr_basic::parse_expr_basic;
6use crate::Parser;
7
8pub fn parse_expr(p: &mut Parser) -> Result<Expr> {
9 let mut expr = parse_expr_basic(p)?;
10
11 // Edge case for return. This has already parsed all the way to the end of the expression
12 if let ExprData::Return(_) = expr.data {
13 return Ok(expr);
14 }
15
16 // Parse operators
17 loop {
18 if let Some(token) = p.peek() {
19 if let Some(level) = BinaryOperator::from_token(&token).map(|o| o.level()) {
20 _ = p.next();
21 expr =
22 parse_expr_binary(p, BinaryOperator::from_token(&token).unwrap(), expr, level)?;
23 continue;
24 }
25 }
26 break;
27 }
28
29 Ok(expr)
30}
31
32/// Prerequisite: `p.peek()` is an operator
33fn parse_expr_binary(
34 p: &mut Parser,
35 op: BinaryOperator,
36 left: Expr,
37 level: OperatorLevel,
38) -> Result<Expr> {
39 let start = p.index;
40 let mut right = parse_expr_basic(p)?;
41
42 loop {
43 if let Some(token) = p.peek() {
44 if let Some(new_level) = BinaryOperator::from_token(&token).map(|o| o.level()) {
45 if (level.associativity() == Associativity::LeftToRight && new_level < level)
46 || (level.associativity() == Associativity::RightToLeft && new_level <= level)
47 {
48 _ = p.next();
49 right = parse_expr_binary(
50 p,
51 BinaryOperator::from_token(&token).unwrap(),
52 right,
53 new_level,
54 )?;
55 continue;
56 }
57 }
58 }
59 break;
60 }
61
62 Ok(Expr::new(
63 ExprData::Binary {
64 op,
65 left: Box::new(left),
66 right: Box::new(right),
67 },
68 Range::new(start, p.index - start),
69 None,
70 ))
71}
crates/parser/parse_expr_basic.rs deleted-230
......@@ -1,230 +0,0 @@
1use ast::{BuiltinId, Expr, ExprData, Literal, UnaryOperator};
2use log::Range;
3use miette::Result;
4use tokenizer::TokenType;
5
6use crate::{
7 parse_block::parse_block, parse_call_arguments::parse_call_arguments, parse_expr::parse_expr,
8 parse_expr_if::parse_expr_if, Parser,
9};
10
11pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {
12 let start = p.index;
13 let comments = p.take_comments();
14 let mut expr = match p.peek() {
15 Some(TokenType::Number(number)) => {
16 _ = p.next();
17 Expr::new(
18 ExprData::Literal(Literal::Number(number.clone())),
19 Range::new(start, p.index - start),
20 comments,
21 )
22 }
23 Some(TokenType::String(string)) => {
24 _ = p.next();
25 Expr::new(
26 ExprData::Literal(Literal::String(string.clone())),
27 Range::new(start, p.index - start),
28 comments,
29 )
30 }
31 Some(TokenType::True) => {
32 _ = p.next();
33 Expr::new(
34 ExprData::Literal(Literal::Bool(true)),
35 Range::new(start, p.index - start),
36 comments,
37 )
38 }
39 Some(TokenType::False) => {
40 _ = p.next();
41 Expr::new(
42 ExprData::Literal(Literal::Bool(false)),
43 Range::new(start, p.index - start),
44 comments,
45 )
46 }
47 Some(TokenType::Void) => {
48 _ = p.next();
49 Expr::new(
50 ExprData::Literal(Literal::Void),
51 Range::new(start, p.index - start),
52 comments,
53 )
54 }
55 Some(TokenType::Identifier(name)) => {
56 let identifier_start = p.index;
57 _ = p.next();
58 if name.starts_with("@") {
59 if let Some(id) = BuiltinId::from_str(name.as_str()) {
60 if let Some(TokenType::OpenParen) = p.next() {
61 let arguments = parse_call_arguments(p)?;
62 Expr::new(
63 ExprData::Builtin { id, arguments },
64 Range::new(identifier_start, p.index - identifier_start),
65 comments,
66 )
67 } else {
68 return Err(p
69 .error_at(
70 format!("Built-in function '{}' must be called", name),
71 Range::new(identifier_start, p.index - identifier_start),
72 )
73 .into());
74 }
75 } else {
76 return Err(p
77 .error_at(
78 format!("Non-existant built-in: '{}'", name),
79 Range::new(identifier_start, p.index - identifier_start),
80 )
81 .into());
82 }
83 } else {
84 Expr::new(
85 ExprData::Identifier(name),
86 Range::new(identifier_start, p.index - identifier_start),
87 comments,
88 )
89 }
90 }
91
92 Some(TokenType::Return) => {
93 if p.is_top_level {
94 return Err(p
95 .error("'return' cannot be used at top level".to_string())
96 .into());
97 } else {
98 _ = p.next();
99 let inner = maybe_parse_expr_basic(p)?.map(Box::new);
100
101 return Ok(Some(Expr::new(
102 ExprData::Return(inner),
103 Range::new(start, p.index - start),
104 comments,
105 )));
106 }
107 }
108
109 Some(TokenType::If) => Expr::new(
110 ExprData::If(parse_expr_if(p)?),
111 Range::new(start, p.index - start),
112 comments,
113 ),
114
115 Some(TokenType::OpenBrace) => Expr::new(
116 ExprData::Block(parse_block(p)?),
117 Range::new(start, p.index - start),
118 comments,
119 ),
120
121 Some(TokenType::OpenParen) => {
122 _ = p.next();
123 let expr = parse_expr(p)?;
124 match p.next() {
125 Some(TokenType::ClosedParen) => (),
126 Some(token) => {
127 return Err(p.error(format!("Expected ')', but got {:?}", token)).into())
128 }
129 None => return Err(p.error("Unexpected end of file".to_string()).into()),
130 }
131 expr
132 }
133
134 Some(t @ TokenType::Bang) | Some(t @ TokenType::Minus) | Some(t @ TokenType::Tilde) => {
135 _ = p.next();
136 Expr::new(
137 ExprData::Unary {
138 op: match t {
139 TokenType::Bang => UnaryOperator::Not,
140 TokenType::Minus => UnaryOperator::Neg,
141 TokenType::Tilde => UnaryOperator::BitNot,
142 _ => unreachable!(),
143 },
144 value: Box::new(parse_expr_basic(p)?),
145 },
146 Range::new(start, p.index - start),
147 comments,
148 )
149 }
150
151 Some(_) | None => return Ok(None),
152 };
153
154 // Parse function calls and field accesses
155 loop {
156 match p.peek() {
157 Some(TokenType::Dot) => {
158 let field_start = p.index;
159 _ = p.next();
160 let field = match p.peek() {
161 Some(TokenType::Identifier(name)) => {
162 let field_start = p.index;
163 p.next();
164 Expr::new(
165 ExprData::Literal(Literal::String(name)),
166 Range::new(field_start, p.index - field_start),
167 None,
168 )
169 }
170 _ => {
171 return Err(p.error("Expected identifier after dot".to_string()).into());
172 }
173 };
174 expr = Expr::new(
175 ExprData::Field {
176 base: Box::new(expr),
177 field: Box::new(field),
178 },
179 Range::new(field_start, p.index - field_start),
180 None,
181 );
182 }
183 Some(TokenType::OpenParen) => {
184 let expr_start = p.index;
185 _ = p.next();
186 let arguments = parse_call_arguments(p)?;
187 expr = Expr::new(
188 ExprData::Call {
189 base: Box::new(expr),
190 arguments,
191 },
192 Range::new(expr_start, p.index - expr_start),
193 None,
194 );
195 }
196 Some(TokenType::OpenBracket) => {
197 let expr_start = p.index;
198 _ = p.next();
199 let index = parse_expr(p)?;
200 if let Some(TokenType::ClosedBracket) = p.next() {
201 expr = Expr::new(
202 ExprData::Field {
203 base: Box::new(expr),
204 field: Box::new(index),
205 },
206 Range::new(expr_start, p.index - expr_start),
207 None,
208 );
209 } else {
210 return Err(p
211 .error("Expected closing bracket after index".to_string())
212 .into());
213 }
214 }
215 _ => break,
216 }
217 }
218
219 Ok(Some(expr))
220}
221
222pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {
223 match maybe_parse_expr_basic(p)? {
224 Some(expr) => Ok(expr),
225 None => match p.peek() {
226 Some(token) => Err(p.error(format!("Unexpected '{}'", token)).into()),
227 None => Err(p.error("Unexpected end of file".to_string()).into()),
228 },
229 }
230}
crates/parser/parse_expr_if.rs deleted-71
......@@ -1,71 +0,0 @@
1use ast::{BlockOrIf, BlockWithRange, If};
2use log::Range;
3use miette::Result;
4use tokenizer::TokenType;
5
6use crate::parse_block::parse_block;
7use crate::parse_expr::parse_expr;
8use crate::Parser;
9
10/// Precondition: the first token exists and is 'if'
11pub fn parse_expr_if(p: &mut Parser) -> Result<If> {
12 match p.next() {
13 Some(TokenType::If) => (),
14 _ => unreachable!(),
15 }
16
17 match p.peek() {
18 Some(TokenType::OpenBrace) => {
19 return Err(p
20 .error(
21 "To use a block as the condition for an if statement, wrap it in parentheses"
22 .to_string(),
23 )
24 .into());
25 }
26 _ => (),
27 }
28
29 let condition = Box::new(parse_expr(p)?);
30
31 let yes = match p.peek() {
32 Some(TokenType::OpenBrace) => {
33 let block_start = p.index;
34 BlockWithRange {
35 block: parse_block(p)?,
36 range: Range::new(block_start, p.index - block_start),
37 }
38 }
39 _ => {
40 return Err(p
41 .error("Expected a block after condition".to_string())
42 .into())
43 }
44 };
45
46 let no = match p.peek() {
47 Some(TokenType::Else) => {
48 _ = p.next();
49 match p.peek() {
50 Some(TokenType::OpenBrace) => {
51 let block_start = p.index;
52 // let comments = p.take_comments();
53 Some(BlockOrIf::Block(BlockWithRange {
54 block: parse_block(p)?,
55 range: Range::new(block_start, p.index - block_start),
56 }))
57 }
58 Some(TokenType::If) => {
59 // TODO(@paperdave): Save location for the if
60 // let range = p.range();
61 // let comments = p.take_comments();
62 Some(BlockOrIf::If(Box::new(parse_expr_if(p)?)))
63 }
64 _ => return Err(p.error("Expected a block after else".to_string()).into()),
65 }
66 }
67 _ => None,
68 };
69
70 Ok(If { condition, yes, no })
71}
crates/parser/parse_fn_declaration.rs deleted-127
......@@ -1,127 +0,0 @@
1use ast::{DocComment, Expr, ExprData, FnDecl, Type};
2use log::Range;
3use miette::Result;
4use tokenizer::TokenType;
5
6use crate::parse_block::parse_block;
7use crate::parse_expr::parse_expr;
8use crate::parse_param::parse_param;
9use crate::parse_type::parse_type;
10use crate::Parser;
11
12/// Precondition:
13pub fn parse_fn_declaration(
14 p: &mut Parser,
15 is_pub: bool,
16 comment: Option<DocComment>,
17) -> Result<FnDecl> {
18 let start = p.index;
19 match p.next() {
20 Some(TokenType::Fn) => (),
21 _ => unreachable!(),
22 }
23
24 let name = match p.next() {
25 Some(TokenType::Identifier(name)) => name,
26 _ => return Err(p.error("Expected identifier after 'fn'".to_string()).into()),
27 };
28
29 match p.peek() {
30 Some(TokenType::OpenParen) => p.next(),
31 _ => {
32 return Err(p
33 .error("Expected '(' after function name".to_string())
34 .into())
35 }
36 };
37
38 let mut params = Vec::new();
39
40 loop {
41 match p.peek() {
42 Some(TokenType::ClosedParen) => {
43 _ = p.next();
44 break;
45 }
46 Some(TokenType::Identifier(_)) => {
47 params.push(parse_param(p)?);
48
49 match p.peek() {
50 Some(TokenType::Comma) => {
51 p.next();
52 if let Some(TokenType::ClosedParen) = p.peek() {
53 p.next();
54 break;
55 }
56 }
57 Some(TokenType::ClosedParen) => {
58 p.next();
59 break;
60 }
61 _ => {
62 let actual = p.peek().map_or("EOF".to_string(), |x| format!("{:?}", x));
63 return Err(p
64 .error(format!(
65 "Expected ',' or ')' after identifier, got {}",
66 actual
67 ))
68 .into());
69 }
70 };
71 }
72 _ => {
73 return Err(p
74 .error("Expected ')' or identifier after '('".to_string())
75 .into())
76 }
77 }
78 }
79
80 let return_type = match p.peek() {
81 Some(TokenType::MinusLt) => {
82 _ = p.next();
83 parse_type(p)?
84 }
85 _ => Type::Infer,
86 };
87
88 let was_in_fn = p.is_in_fn;
89 p.is_in_fn = true;
90
91 let body = match p.peek() {
92 Some(TokenType::OpenBrace) => {
93 let block_start = p.index;
94 let block = parse_block(p)?;
95 Expr::new(
96 ExprData::Block(block),
97 Range::new(block_start, p.index - block_start),
98 None,
99 )
100 }
101 Some(TokenType::EqualLt) => {
102 _ = p.next();
103 parse_expr(p)?
104 }
105 _ => {
106 let actually = p.peek().map_or("EOF".to_string(), |x| format!("{:?}", x));
107 return Err(p
108 .error(format!(
109 "Expected a block or '=>' to follow function declaration, but got {}",
110 actually,
111 ))
112 .into());
113 }
114 };
115
116 p.is_in_fn = was_in_fn;
117
118 Ok(FnDecl {
119 name,
120 return_type,
121 params,
122 body,
123 is_pub,
124 comment,
125 range: Range::new(start, p.index - start),
126 })
127}
crates/parser/parse_module.rs deleted-44
......@@ -1,44 +0,0 @@
1use crate::parse_doc_comment::maybe_read_doc_comment;
2use crate::parse_statement::parse_statement;
3use crate::Parser;
4use ast::Module;
5use miette::Result;
6use tokenizer::TokenType;
7
8pub fn parse_module(p: &mut Parser) -> Result<Module> {
9 let mut statements = Vec::new();
10
11 let file_doc_comment = maybe_read_doc_comment(p);
12
13 loop {
14 let is_statement_like_expression = p
15 .peek()
16 .map_or(false, |x| x.is_start_of_statement_like_expression());
17
18 let statement = parse_statement(p)?;
19
20 match p.peek() {
21 Some(TokenType::Semicolon) => {
22 while let Some(TokenType::Semicolon) = p.peek() {
23 p.next();
24 }
25 }
26 _ => {
27 if statement.require_semicolon(is_statement_like_expression) {
28 return Err(p
29 .error("Expected semicolon after statement".to_string())
30 .into());
31 }
32 }
33 };
34
35 statements.push(statement);
36
37 if p.empty() {
38 return Ok(Module {
39 statements,
40 comment: file_doc_comment,
41 });
42 }
43 }
44}
crates/parser/parse_param.rs deleted-30
......@@ -1,30 +0,0 @@
1use ast::Param;
2use miette::Result;
3use tokenizer::TokenType;
4
5use crate::{parse_type::parse_type, Parser};
6
7/// Precondition: A token exists and is an identifier
8pub fn parse_param(p: &mut Parser) -> Result<Param> {
9 let name = match p.next() {
10 Some(TokenType::Identifier(name)) => name.clone(),
11 _ => unreachable!(),
12 };
13
14 let param_type = match p.peek() {
15 Some(TokenType::Colon) => {
16 _ = p.next();
17 parse_type(p)?
18 }
19 _ => {
20 return Err(p
21 .error("Function parameters require type annotations".to_string())
22 .into())
23 }
24 };
25
26 Ok(Param {
27 name: name.clone(),
28 param_type,
29 })
30}
crates/parser/parse_statement.rs deleted-88
......@@ -1,88 +0,0 @@
1use ast::Statement;
2use miette::Result;
3use tokenizer::TokenType;
4
5use crate::parse_declaration::parse_declaration;
6use crate::parse_doc_comment::maybe_read_doc_comment;
7use crate::parse_expr::parse_expr;
8use crate::parse_fn_declaration::parse_fn_declaration;
9use crate::Parser;
10
11pub fn parse_statement(p: &mut Parser) -> Result<Statement> {
12 if let Some(comments) = p.take_comments() {
13 return Ok(Statement::Comment(comments));
14 }
15
16 let m_doc_comment = maybe_read_doc_comment(p);
17 if let Some(doc_comment) = m_doc_comment.clone() {
18 if p.has_comments() {
19 // this implies that comments are immediately after the doc
20 return Err(p
21 .error_at(format!("Unattached doc comment"), doc_comment.range)
22 .into());
23 }
24 }
25
26 match p.peek() {
27 Some(TokenType::DocComment(_)) => unreachable!(),
28 Some(TokenType::Pub) => {
29 if !p.is_top_level {
30 return Err(p
31 .error("Public declarations are only allowed at the top level".to_string())
32 .into());
33 }
34 _ = p.next();
35 match p.peek() {
36 Some(TokenType::Var) | Some(TokenType::Let) => Ok(Statement::VarDecl(
37 parse_declaration(p, true, m_doc_comment)?,
38 )),
39 Some(TokenType::Fn) => Ok(Statement::FnDecl(parse_fn_declaration(
40 p,
41 true,
42 m_doc_comment,
43 )?)),
44 Some(_) => Err(p.error("Expected a declaration".to_string()).into()),
45 None => Err(p.error("Unexpected end of file".to_string()).into()),
46 }
47 }
48 Some(TokenType::Var) | Some(TokenType::Let) => Ok(Statement::VarDecl(parse_declaration(
49 p,
50 false,
51 m_doc_comment,
52 )?)),
53 Some(TokenType::Fn) => Ok(Statement::FnDecl(parse_fn_declaration(
54 p,
55 false,
56 m_doc_comment,
57 )?)),
58 Some(TokenType::Defer) => {
59 let label = p.peek().unwrap();
60 Err(p
61 .error(format!("'{}' is not yet implemented", label))
62 .into())
63 }
64 Some(TokenType::Struct | TokenType::Enum | TokenType::Trait | TokenType::Impl) => {
65 let label = p.peek().unwrap();
66 if !p.is_top_level {
67 return Err(p
68 .error(format!("'{}' is only allowed at the top level", label))
69 .into());
70 }
71
72 Err(p
73 .error(format!("'{}' is not yet implemented", label))
74 .into())
75 }
76
77 Some(_) => {
78 if let Some(doc_comment) = m_doc_comment {
79 Err(p
80 .error_at(format!("Unattached doc comment"), doc_comment.range)
81 .into())
82 } else {
83 Ok(Statement::Expr(parse_expr(p)?))
84 }
85 }
86 None => Err(p.error("Unexpected end of file".to_string()).into()),
87 }
88}
crates/parser/parse_type.rs deleted-26
......@@ -1,26 +0,0 @@
1use ast::Type;
2use miette::Result;
3
4use tokenizer::TokenType;
5
6use crate::Parser;
7
8pub fn parse_type(p: &mut Parser) -> Result<Type> {
9 match p.next() {
10 Some(TokenType::Identifier(id)) => match id.as_str() {
11 "int" => Ok(Type::Int),
12 "float" => Ok(Type::Float),
13 "bigint" => Ok(Type::BigInt),
14 "bool" => Ok(Type::Bool),
15 "string" => Ok(Type::String),
16 invalid_type => Err(p
17 .error(format!(
18 "Invalid type '{}' (TODO: add custom type support)",
19 invalid_type
20 ))
21 .into()),
22 },
23 Some(unexpected) => Err(p.error(format!("Expected type, got {}", unexpected)).into()),
24 None => Err(p.error("Expected type, got EOF".to_string()).into()),
25 }
26}
crates/parser/parser.rs deleted-150
......@@ -1,150 +0,0 @@
1mod parse_block;
2mod parse_call_arguments;
3mod parse_declaration;
4mod parse_doc_comment;
5mod parse_expr;
6mod parse_expr_basic;
7mod parse_expr_if;
8mod parse_fn_declaration;
9mod parse_module;
10mod parse_param;
11mod parse_statement;
12mod parse_type;
13
14use ast::Comment;
15use ast::Module;
16use log::Range;
17use miette::Diagnostic;
18use miette::Result;
19use parse_module::parse_module;
20use thiserror::Error;
21use tokenizer::tokenize;
22use tokenizer::Token;
23use tokenizer::TokenType;
24
25pub fn parse(tokens: Vec<Token>) -> Result<Module> {
26 let mut p = Parser::new(tokens);
27 parse_module(&mut p)
28}
29
30pub fn parse_from_string(string: String) -> Result<Module> {
31 let tokens = tokenize(string)?;
32 parse(tokens)
33}
34
35#[derive(Error, Diagnostic, Debug)]
36#[error("ParseError")]
37pub struct ParseError {
38 #[help]
39 error: String,
40 range: Option<Range>,
41}
42
43impl ParseError {
44 pub fn new(error: String, range: Option<Range>) -> ParseError {
45 ParseError { error, range }
46 }
47}
48
49#[derive(Debug)]
50pub struct Parser {
51 /// TODO(@paperdave): switch to streaming tokenizer + one that can
52 /// report line and column info
53 tokens: Vec<Token>,
54 cursor: usize,
55
56 /// Current index in the string
57 pub index: usize,
58
59 /// Whether or not the parser is at the top level.
60 /// Currently controls the validity of `pub` keywords.
61 pub is_top_level: bool,
62 /// Whether or not the parser is inside a function.
63 /// Controls the validity of the `return` keyword.
64 pub is_in_fn: bool,
65
66 comments: Vec<String>,
67}
68
69impl Parser {
70 pub fn new(tokens: Vec<Token>) -> Parser {
71 Parser {
72 tokens,
73 cursor: 0,
74 index: 0,
75 is_top_level: true,
76 is_in_fn: false,
77 comments: vec![],
78 }
79 }
80
81 pub fn empty(&self) -> bool {
82 self.cursor == self.tokens.len()
83 }
84
85 pub fn next(&mut self) -> Option<TokenType> {
86 if self.cursor == self.tokens.len() {
87 return None;
88 }
89
90 loop {
91 let token = self.tokens[self.cursor].clone();
92 self.index = token.range.start;
93 match token.variant {
94 TokenType::Comment(comment) => {
95 self.comments.push(comment);
96 self.cursor += 1;
97 continue;
98 }
99 token => {
100 self.cursor += 1;
101 return Some(token);
102 }
103 }
104 }
105 }
106
107 pub fn take_comments(&mut self) -> Option<Comment> {
108 if self.comments.len() == 0 {
109 return None;
110 }
111 let comments = self.comments.clone();
112 self.comments = vec![];
113 Some(Comment {
114 data: comments.join("\n"),
115 })
116 }
117
118 pub fn has_comments(&mut self) -> bool {
119 self.peek();
120 self.comments.len() > 0
121 }
122
123 pub fn peek(&mut self) -> Option<TokenType> {
124 loop {
125 if self.cursor == self.tokens.len() {
126 return None;
127 }
128
129 match self.tokens[self.cursor].clone().variant {
130 TokenType::Comment(comment) => {
131 self.comments.push(comment);
132 self.cursor += 1;
133 continue;
134 }
135 token => {
136 return Some(token);
137 }
138 }
139 }
140 }
141
142 pub fn error(&self, error: String) -> ParseError {
143 // TODO(@lino-levan): Add location information
144 ParseError::new(error, None)
145 }
146
147 fn error_at(&self, error: String, range: Range) -> ParseError {
148 ParseError::new(error, Some(range))
149 }
150}
crates/sema/Cargo.toml deleted-14
......@@ -1,14 +0,0 @@
1[package]
2name = "sema"
3version = "0.1.0"
4edition = "2021"
5
6[lib]
7path = "./sema.rs"
8
9[dependencies]
10miette.workspace = true
11thiserror.workspace = true
12num.workspace = true
13
14ast = { path = "../ast" }
\ No newline at end of file
crates/sema/ast_type_coersion.rs deleted-54
......@@ -1,54 +0,0 @@
1// use ast::{BinaryOperator, ExprData, Type};
2
3impl TypeCoerceable for ExprData {
4 fn to_type(&self) -> Option<Type> {
5 match self {
6 ExprData::Block(block) => {
7 if (block.has_implicit_return) {
8 block.statements.last().unwrap().to_type()
9 } else {
10 Some(Type::Void)
11 }
12 }
13 ExprData::Binary { op, left, right } => {
14 // TODO: this likely has to do further type checking
15 let left_type = left.to_type();
16 match op {
17 BinaryOperator::Add => left_type,
18 BinaryOperator::Sub => left_type,
19 BinaryOperator::Mul => left_type,
20 BinaryOperator::Div => left_type,
21 BinaryOperator::Pow => left_type,
22 BinaryOperator::Eq => Some(Type::Bool),
23 BinaryOperator::Gt => Some(Type::Bool),
24 BinaryOperator::Lt => Some(Type::Bool),
25 BinaryOperator::Gte => Some(Type::Bool),
26 BinaryOperator::Lte => Some(Type::Bool),
27 BinaryOperator::Lshift => Some(Type::Int),
28 BinaryOperator::Rshift => Some(Type::Int),
29 BinaryOperator::BitAnd => Some(Type::Int),
30 BinaryOperator::BitOr => Some(Type::Int),
31 BinaryOperator::BitXor => Some(Type::Int),
32 BinaryOperator::Mod => Some(Type::Int),
33 BinaryOperator::NotEq => Some(Type::Int),
34
35 BinaryOperator::And => Some(Type::Bool),
36 BinaryOperator::Or => Some(Type::Bool),
37 }
38 }
39 ExprData::Unary { op, value } => todo!(),
40 ExprData::Function {
41 name,
42 params,
43 return_type,
44 block,
45 } => Some(return_type),
46 ExprData::Call { base, arguments } => None, // oh shit. yeah this trait isn't implementable
47 ExprData::Builtin { id, arguments } => todo!(),
48 ExprData::If(_) => todo!(),
49 ExprData::Return(_) => todo!(),
50 ExprData::Literal(_) => todo!(),
51 ExprData::Identifier(_) => todo!(),
52 }
53 }
54}
crates/sema/sema.rs deleted-98
......@@ -1,98 +0,0 @@
1use ast::Module;
2use miette::Result;
3
4pub fn analyze_module(_block: &mut Module) -> Result<()> {
5 //
6 Ok(())
7}
8
9// for statement in &block.statements {
10// check_expression(statement)?;
11// }
12
13// Ok(())
14// }
15
16// pub fn resolve_type(expr: &Ast) -> Result<Type> {
17// match expr {
18// Ast::Number(value) => Ok(Type::NumberLiteral(value.clone())),
19// Ast::Declaration {
20// name,
21// decl_type,
22// value,
23// is_constant,
24// } => Ok(Type::NoReturn),
25// Ast::Binary {
26// operator,
27// left,
28// right,
29// } => merge_types(resolve_type(left)?, resolve_type(right)?),
30// Ast::Function {
31// name,
32// arguments,
33// return_type,
34// block,
35// } => todo!(),
36// Ast::FunctionCall { name, arguments } => todo!(),
37// Ast::If {
38// condition,
39// value,
40// else_value,
41// } => todo!(),
42// Ast::Return(_) => Ok(Type::NoReturn),
43// Ast::Public(_) => todo!(), // TODO: remove Public
44// Ast::Block(_) => todo!(),
45// Ast::Bool(_) => Ok(Type::Bool),
46// Ast::String(_) => Ok(Type::String),
47// Ast::Identifier(_) => todo!(),
48// Ast::BuiltIn(_) => todo!(),
49
50// Ast::Comment(_) => unreachable!(),
51// Ast::DocComment(_) => unreachable!(),
52// }
53// }
54
55// pub fn check_expression(expr: &Ast) -> Result<()> {
56// panic!()
57// match expr {
58// Ast::Declaration {
59// name,
60// decl_type: return_type,
61// value,
62// is_constant,
63// } => {
64// check_expression(value)?;
65// // coerce_type(return_type.clone(), resolve_type(value)?)?;
66// }
67// _ => {}
68// // Ast::Operation {
69// // operator,
70// // left,
71// // right,
72// // } => Ok(()),
73// // Ast::Function {
74// // name,
75// // arguments,
76// // return_type,
77// // block,
78// // } => Ok(()),
79// // Ast::FunctionCall { name, arguments } => Ok(()),
80// // Ast::If {
81// // condition,
82// // value,
83// // else_value,
84// // } => Ok(()),
85// // Ast::Return(_) => Ok(()),
86// // Ast::Public(_) => Ok(()),
87// // Ast::Block(_) => Ok(()),
88// // Ast::Number(_) => Ok(()),
89// // Ast::Bool(_) => Ok(()),
90// // Ast::String(_) => Ok(()),
91// // Ast::Identifier(_) => Ok(()),
92// // Ast::BuiltIn(_) => Ok(()),
93// // Ast::Comment(_) => Ok(()),
94// // Ast::DocComment(_) => Ok(()),
95// }
96
97// Ok(())
98// }
crates/sema/sema_scope.rs deleted
crates/tokenizer/Cargo.toml deleted-13
......@@ -1,13 +0,0 @@
1[package]
2name = "tokenizer"
3version = "0.1.0"
4edition = "2021"
5
6[lib]
7path="./tokenizer.rs"
8
9[dependencies]
10miette.workspace = true
11thiserror.workspace = true
12
13log = { path = "../log" }
crates/tokenizer/tokenizer.rs deleted-443
......@@ -1,443 +0,0 @@
1use log::Range;
2use miette::{Diagnostic, Result};
3use thiserror::Error;
4
5#[derive(Error, Diagnostic, Debug)]
6#[error("UnknownCharacter")]
7struct UnknownCharacter {
8 #[help]
9 error: String,
10}
11
12#[derive(Debug, Clone, PartialEq)]
13pub enum TokenType {
14 // Literals
15 Identifier(String),
16 Number(f64),
17 String(String),
18 // Comments
19 Comment(String),
20 DocComment(String),
21 // Keywords
22 Defer,
23 Else,
24 Enum,
25 False,
26 Fn,
27 If,
28 Impl,
29 Let,
30 Pub,
31 Return,
32 Struct,
33 Trait,
34 True,
35 Var,
36 Void,
37 // Symbols
38 And, // &
39 AndAnd, // &&
40 Bang, // !
41 BangEqual, // !=
42 EqualLt, // =>
43 Caret, // ^
44 ClosedBrace, // }
45 ClosedBracket, // ]
46 ClosedParen, // )
47 Colon, // :
48 Comma, // ,
49 Dot, // .
50 Equal, // =
51 EqualEqual, // ==
52 Gt, // >
53 GtEqual, // >=
54 GtGt, // >>
55 GtGtGt, // >>>
56 Lt, // <
57 LtEqual, // <=
58 LtLt, // <<
59 Minus, // -
60 OpenBrace, // {
61 OpenBracket, // [
62 OpenParen, // (
63 Or, // |
64 OrOr, // ||
65 Percent, // %
66 Plus, // +
67 MinusLt, // ->
68 Semicolon, // ;
69 Slash, // /
70 Star, // *
71 StarStar, // **
72 Tilde, // ~
73}
74
75impl TokenType {
76 pub fn is_start_of_statement_like_expression(&self) -> bool {
77 match &self {
78 TokenType::If | TokenType::OpenBrace => true,
79 _ => false,
80 }
81 }
82}
83
84impl std::fmt::Display for TokenType {
85 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
86 let token = match &self {
87 TokenType::Identifier(_) => "identifier".to_string(),
88
89 TokenType::Number(n) => n.to_string(),
90 TokenType::String(s) => format!("\"{}\"", s),
91 TokenType::Comment(_) => "comment".to_string(),
92 TokenType::DocComment(_) => "documentation comment".to_string(),
93
94 TokenType::Defer => "defer".to_string(),
95 TokenType::Else => "else".to_string(),
96 TokenType::Enum => "enum".to_string(),
97 TokenType::False => "false".to_string(),
98 TokenType::Fn => "fn".to_string(),
99 TokenType::If => "if".to_string(),
100 TokenType::Impl => "impl".to_string(),
101 TokenType::Let => "let".to_string(),
102 TokenType::Pub => "pub".to_string(),
103 TokenType::Return => "return".to_string(),
104 TokenType::Struct => "struct".to_string(),
105 TokenType::Trait => "trait".to_string(),
106 TokenType::True => "true".to_string(),
107 TokenType::Var => "var".to_string(),
108 TokenType::Void => "void".to_string(),
109
110 TokenType::And => "&".to_string(),
111 TokenType::AndAnd => "&&".to_string(),
112 TokenType::Bang => "!".to_string(),
113 TokenType::BangEqual => "!=".to_string(),
114 TokenType::EqualLt => "=>".to_string(),
115 TokenType::Caret => "^".to_string(),
116 TokenType::ClosedBrace => "}".to_string(),
117 TokenType::ClosedBracket => "]".to_string(),
118 TokenType::ClosedParen => ")".to_string(),
119 TokenType::Colon => ":".to_string(),
120 TokenType::Comma => ",".to_string(),
121 TokenType::Dot => ".".to_string(),
122 TokenType::Equal => "=".to_string(),
123 TokenType::EqualEqual => "==".to_string(),
124 TokenType::Gt => ">".to_string(),
125 TokenType::GtEqual => ">=".to_string(),
126 TokenType::GtGt => ">>".to_string(),
127 TokenType::GtGtGt => ">>>".to_string(),
128 TokenType::Lt => "<".to_string(),
129 TokenType::LtEqual => "<=".to_string(),
130 TokenType::LtLt => "<<".to_string(),
131 TokenType::Minus => "-".to_string(),
132 TokenType::OpenBrace => "{".to_string(),
133 TokenType::OpenBracket => "[".to_string(),
134 TokenType::OpenParen => "(".to_string(),
135 TokenType::Or => "|".to_string(),
136 TokenType::OrOr => "||".to_string(),
137 TokenType::Percent => "%".to_string(),
138 TokenType::Plus => "+".to_string(),
139 TokenType::MinusLt => "->".to_string(),
140 TokenType::Semicolon => ";".to_string(),
141 TokenType::Slash => "/".to_string(),
142 TokenType::Star => "*".to_string(),
143 TokenType::StarStar => "**".to_string(),
144 TokenType::Tilde => "~".to_string(),
145 };
146
147 write!(f, "{}", token)
148 }
149}
150
151#[derive(Debug, Clone)]
152pub struct Token {
153 pub variant: TokenType,
154 pub range: Range,
155}
156
157impl Token {
158 pub fn new(variant: TokenType, start: usize, length: usize) -> Self {
159 Self {
160 variant,
161 range: Range::new(start, length),
162 }
163 }
164}
165
166// TODO(@paperdave): rewrite this to not use a Vec<Token>. we probably dont need it. also store Range
167pub fn tokenize(content: String) -> Result<Vec<Token>> {
168 let mut tokens: Vec<Token> = Vec::new();
169 let mut chars = content.chars().enumerate().peekable();
170
171 loop {
172 match chars.next() {
173 Some((start, c)) => {
174 let token = match c {
175 'a'..='z' | 'A'..='Z' | '_' | '@' => {
176 let mut current_token = String::new();
177 current_token.push(c);
178 loop {
179 let c = chars.peek();
180 if c.is_none() {
181 break;
182 }
183 let (_, c) = c.unwrap();
184 match c {
185 'a'..='z' | 'A'..='Z' | '_' | '0'..='9' => {
186 current_token.push(*c);
187 chars.next();
188 }
189 _ => break,
190 }
191 }
192
193 let length = current_token.len();
194
195 let variant = match current_token.as_str() {
196 "defer" => TokenType::Defer,
197 "else" => TokenType::Else,
198 "enum" => TokenType::Enum,
199 "false" => TokenType::False,
200 "fn" => TokenType::Fn,
201 "if" => TokenType::If,
202 "impl" => TokenType::Impl,
203 "let" => TokenType::Let,
204 "pub" => TokenType::Pub,
205 "return" => TokenType::Return,
206 "struct" => TokenType::Struct,
207 "trait" => TokenType::Trait,
208 "true" => TokenType::True,
209 "var" => TokenType::Var,
210 "void" => TokenType::Void,
211 _ => TokenType::Identifier(current_token),
212 };
213
214 Token::new(variant, start, length)
215 }
216 '"' => {
217 let mut length = 1;
218 let mut current_token = String::new();
219 loop {
220 let c = chars.next();
221 if c.is_none() {
222 break;
223 }
224 let (_, c) = c.unwrap();
225 length += 1;
226 match c {
227 '"' => break,
228 _ => current_token.push(c),
229 }
230 }
231 Token::new(TokenType::String(current_token), start, length)
232 }
233 '0'..='9' => {
234 let mut current_token = String::new();
235 current_token.push(c);
236 loop {
237 let c = chars.peek();
238 if c.is_none() {
239 break;
240 }
241 let (_, c) = c.unwrap();
242 match c {
243 '0'..='9' => {
244 current_token.push(*c);
245 chars.next();
246 }
247 _ => break,
248 }
249 }
250 match current_token.parse::<f64>() {
251 Ok(v) => Token::new(TokenType::Number(v), start, current_token.len()),
252 Err(_) => {
253 return Err(UnknownCharacter {
254 error: format!("Unknown character: {}", current_token),
255 }
256 .into())
257 }
258 }
259 }
260 ':' => Token::new(TokenType::Colon, start, 1),
261 '=' => match chars.peek() {
262 Some((_, '=')) => {
263 chars.next();
264 Token::new(TokenType::EqualEqual, start, 2)
265 }
266 Some((_, '>')) => {
267 chars.next();
268 Token::new(TokenType::EqualLt, start, 2)
269 }
270 _ => Token::new(TokenType::Equal, start, 1),
271 },
272 ';' => Token::new(TokenType::Semicolon, start, 1),
273 '>' => match chars.peek() {
274 Some((_, '=')) => {
275 chars.next();
276 Token::new(TokenType::GtEqual, start, 2)
277 }
278 Some((_, '>')) => {
279 chars.next();
280 match chars.peek() {
281 Some((_, '>')) => {
282 chars.next();
283 Token::new(TokenType::GtGtGt, start, 3)
284 }
285 _ => Token::new(TokenType::GtGt, start, 2),
286 }
287 }
288 _ => Token::new(TokenType::Gt, start, 1),
289 },
290 '<' => match chars.peek() {
291 Some((_, '=')) => {
292 chars.next();
293 Token::new(TokenType::LtEqual, start, 2)
294 }
295 Some((_, '<')) => {
296 chars.next();
297 Token::new(TokenType::LtLt, start, 2)
298 }
299 _ => Token::new(TokenType::Lt, start, 1),
300 },
301 '+' => Token::new(TokenType::Plus, start, 1),
302 '-' => match chars.peek() {
303 Some((_, '>')) => {
304 chars.next();
305 Token::new(TokenType::MinusLt, start, 2)
306 }
307 _ => Token::new(TokenType::Minus, start, 1),
308 },
309 '*' => match chars.peek() {
310 Some((_, '*')) => {
311 chars.next();
312 Token::new(TokenType::StarStar, start, 2)
313 }
314 _ => Token::new(TokenType::Star, start, 1),
315 },
316 '/' => {
317 match chars.peek() {
318 Some((_, '/')) => {
319 chars.next(); // Consume the second slash
320 let mut length = 2;
321
322 let is_doc_comment = match chars.peek() {
323 Some((_, '/')) => {
324 chars.next(); // Consume the third slash
325 length += 1;
326 true
327 }
328 _ => false,
329 };
330
331 let mut comment = String::new();
332 loop {
333 // skip leading whitespace
334 while let Some((_, c)) = chars.peek() {
335 if !c.is_whitespace() {
336 break;
337 }
338 chars.next();
339 length += 1;
340 }
341 // read the comment
342 while let Some((_, c)) = chars.next() {
343 length += 1;
344 if c == '\n' || c == '\r' {
345 break;
346 }
347 comment.push(c);
348 }
349 // trim trailing whitespace on line
350 comment = comment.trim_end().to_string();
351 // skip to next non-whitespace character
352 while let Some((_, c)) = chars.peek() {
353 if !c.is_whitespace() {
354 break;
355 }
356 chars.next();
357 length += 1;
358 }
359 // check if we've hit another comment
360 let mut tmp = chars.clone();
361
362 let is_another_comment = match tmp.peek() {
363 Some((_, '/')) => {
364 tmp.next(); // Consume the second slash
365 match tmp.peek() {
366 Some((_, '/')) => {
367 tmp.next(); // Consume the third slash
368 is_doc_comment
369 }
370 _ => !is_doc_comment,
371 }
372 }
373 _ => false,
374 };
375
376 if is_another_comment {
377 // apply our temporary iterator to the main iterator
378 comment.push('\n');
379 chars = tmp;
380 chars.next();
381 length += 1;
382 } else {
383 break;
384 }
385 }
386
387 if is_doc_comment {
388 Token::new(TokenType::DocComment(comment), start, length)
389 } else {
390 Token::new(TokenType::Comment(comment), start, length)
391 }
392 }
393 _ => Token::new(TokenType::Slash, start, 1),
394 }
395 }
396 ',' => Token::new(TokenType::Comma, start, 1),
397 '(' => Token::new(TokenType::OpenParen, start, 1),
398 ')' => Token::new(TokenType::ClosedParen, start, 1),
399 '{' => Token::new(TokenType::OpenBrace, start, 1),
400 '}' => Token::new(TokenType::ClosedBrace, start, 1),
401 '[' => Token::new(TokenType::OpenBracket, start, 1),
402 ']' => Token::new(TokenType::ClosedBracket, start, 1),
403 '&' => match chars.peek() {
404 Some((_, '&')) => {
405 chars.next();
406 Token::new(TokenType::AndAnd, start, 2)
407 }
408 _ => Token::new(TokenType::And, start, 1),
409 },
410 '|' => match chars.peek() {
411 Some((_, '|')) => {
412 chars.next();
413 Token::new(TokenType::OrOr, start, 2)
414 }
415 _ => Token::new(TokenType::Or, start, 1),
416 },
417 '!' => match chars.peek() {
418 Some((_, '=')) => {
419 chars.next();
420 Token::new(TokenType::BangEqual, start, 2)
421 }
422 _ => Token::new(TokenType::Bang, start, 1),
423 },
424 '^' => Token::new(TokenType::Caret, start, 1),
425 '%' => Token::new(TokenType::Percent, start, 1),
426 '~' => Token::new(TokenType::Tilde, start, 1),
427 '.' => Token::new(TokenType::Dot, start, 1),
428 s if s.is_whitespace() => continue,
429 _ => {
430 return Err(UnknownCharacter {
431 error: format!("Unexpected '{}' while tokenizing", c),
432 }
433 .into())
434 }
435 };
436 tokens.push(token);
437 }
438 None => break,
439 }
440 }
441
442 Ok(tokens)
443}
examples/basics/associativity.blight deleted-6
......@@ -1,6 +0,0 @@
1let print = @raw("console.log");
2print(1 * 2 * 3);
3print(1 * 2 / 3);
4print(1 + 2 ** 3);
5print(1 ** 2 ** 3);
6print(1 * 2 + 3);
examples/basics/comments.blight deleted-9
......@@ -1,9 +0,0 @@
1// this is a regular comment
2let x: int = 0;
3
4/// this is a doc comment
5let y: int = 0;
6
7/// Doc comments can be multiple lines.
8/// This can be done by simply adding another triple-slash to the next line.
9let z: int = 0;
examples/basics/control.blight deleted-17
......@@ -1,17 +0,0 @@
1let print = @raw("console.log");
2
3let x: bool = true;
4
5if x {
6 print("x is true!");
7}
8
9let y: int = 3;
10
11if y == 1 {
12 print("y is 1!");
13} else if y == 2 {
14 print("y is 2!");
15} else {
16 print("y is not 1 or 2!");
17}
examples/basics/defer.blight deleted-12
......@@ -1,12 +0,0 @@
1let print = @raw("console.log");
2
3{
4 print("1");
5 defer print("6");
6 {
7 print("2");
8 defer print("4");
9 print("3");
10 }
11 print("5");
12}
examples/basics/expressions.blight deleted-11
......@@ -1,11 +0,0 @@
1// everything is an expression in blight
2// everything.
3// hehe
4
5let x: int = 5 + {
6 let y: int = 5;
7 y
8};
9
10let print = @raw("console.log");
11print(x); // 10
examples/basics/fib.blight deleted-12
......@@ -1,12 +0,0 @@
1pub fn fib(n: int) -> int {
2 if n == 0 {
3 0
4 } else if n == 1 {
5 1
6 } else {
7 fib(n - 1) + fib(n - 2)
8 }
9}
10
11let print = @raw("console.log");
12print(fib(10));
examples/basics/functions.blight deleted-8
......@@ -1,8 +0,0 @@
1fn bingus(x: int, y: int) -> int {
2 x * y
3}
4
5let result: int = bingus(10, 25);
6
7let print = @raw("console.log");
8print(result);
examples/basics/operations.blight deleted-4
......@@ -1,4 +0,0 @@
1let a: int = 10 - 2 / 4 * 15;
2
3let print = @raw("console.log");
4print(a);
examples/basics/shadowing.blight deleted-14
......@@ -1,14 +0,0 @@
1let print = @raw("console.log");
2
3let x: int = 0;
4let x: int = 1;
5print(x); // 1
6
7let y: int = 0;
8{
9 let y: int = 1;
10 {
11 let y: int = 2;
12 }
13}
14print(y); // 0
examples/basics/stupid.blight deleted-2
......@@ -1,2 +0,0 @@
1
2fn add(x: int, y: int) => x + y
examples/basics/variables.blight deleted-10
......@@ -1,10 +0,0 @@
1var x: int = 0;
2var y: float = 0;
3var z: bigint = 0;
4
5let a: int = 0;
6let b: float = 0;
7let c: bigint = 0;
8
9let print = @raw("console.log");
10print(x);
examples/basics/weird.blight deleted-14
......@@ -1,14 +0,0 @@
1// Just trying to break the blight compiler
2fn bingus(y: bool) -> int {
3 let x: int = 5 + {
4 if y {
5 return 100000;
6 }
7 6
8 };
9 x
10}
11
12let print = @raw("console.log");
13print(bingus(true));
14print(bingus(false));
examples/interop/blight_in_js/add.blight deleted-5
......@@ -1,5 +0,0 @@
1pub fn add(x: int, y: int) -> int {
2 x + y
3}
4
5pub let TOP_LEVEL: int = 100;
examples/interop/blight_in_js/main.js deleted-4
......@@ -1,4 +0,0 @@
1import { add, TOP_LEVEL } from "./add.js";
2
3console.log(add(1, 2));
4console.log(TOP_LEVEL);
examples/interop/deno.blight deleted-7
......@@ -1,7 +0,0 @@
1let writeTextFile = @raw("Deno.writeTextFile");
2
3fn x() -> string {
4 "hello"
5}
6
7writeTextFile("hello.txt", x());
src/compiler/Air.Dump.zig created+299
......@@ -0,0 +1,299 @@
1//! This implements a debug formatter for Teg
2air: Air,
3ip: *const InternPool,
4
5const State = struct {
6 arena: std.mem.Allocator,
7 air: *const Air,
8 ip: *const InternPool,
9 indent: usize = 0,
10 tty: std.io.tty.Config,
11 visited: std.AutoHashMapUnmanaged(Inst.Index, void) = .{},
12
13 fn newline(state: State, writer: anytype) !void {
14 try writer.writeByte('\n');
15 if (state.indent > 0)
16 try writer.writeByteNTimes(' ', state.indent * 2);
17 }
18
19 fn dumpInst(s: *State, w: anytype, inst: Inst.Index) @TypeOf(w).Error!void {
20 const air = s.air;
21 const tty = s.tty;
22 const tag = s.air.instTag(inst);
23
24 try tty.setColor(w, switch (s.air.instFlags(inst).uses) {
25 .none => .bright_black,
26 .multi => .red,
27 .single => .magenta,
28 });
29 try w.writeByte(switch (s.air.instFlags(inst).uses) {
30 .none => 'X',
31 .multi => 'M',
32 .single => 'S',
33 });
34 try tty.setColor(w, .cyan);
35
36 try w.print(" #{} = ", .{
37 @intFromEnum(inst),
38 });
39
40 try tty.setColor(w, .cyan);
41 try w.print("{s}", .{@tagName(tag)});
42 try tty.setColor(w, .reset);
43
44 if ((try s.visited.fetchPut(s.arena, inst, {})) != null) {
45 try tty.setColor(w, .red);
46 try w.writeAll(" [error: recursive tree]");
47 try tty.setColor(w, .reset);
48 return;
49 }
50
51 switch (tag) {
52 .block => {
53 const index_list: Air.IndexList = air.instData(.block, inst);
54 const insts = index_list.slice(air);
55 try s.dumpInstList(w, insts, '{', '}');
56 },
57 .js_global_ref => {
58 const name: InternPool.String.WithLen = air.instData(.js_global_ref, inst);
59 try tty.setColor(w, .magenta);
60 try w.print(" {s}", .{name.toSlice(s.ip)});
61 try tty.setColor(w, .reset);
62 },
63 .arg => {
64 const arg: Air.Arg = air.instData(.arg, inst);
65 try w.writeAll(" ");
66 try tty.setColor(w, .yellow);
67 try w.print("{d}", .{arg.arg_number});
68 try tty.setColor(w, .reset);
69 try w.writeAll(": ");
70 try s.dumpValue(w, arg.type);
71 },
72 .@"break" => {
73 const brk: Air.Break = air.instData(.@"break", inst);
74 try w.writeAll("(");
75 try s.dumpInstRef(w, brk.block_ref);
76 try w.writeAll(", ");
77 try s.dumpRef(w, brk.value);
78 try w.writeAll(")");
79 },
80 inline .add,
81 .mul,
82 .sub,
83 .div,
84 .lt,
85 .gt,
86 .eq,
87 => |comptime_tag| {
88 const bin: Air.Bin = air.instData(comptime_tag, inst);
89 try w.writeAll("(");
90 try s.dumpRef(w, bin.lhs);
91 try w.writeAll(", ");
92 try s.dumpRef(w, bin.rhs);
93 try w.writeAll(")");
94 },
95 inline .negate => |comptime_tag| {
96 const bin: Air.Un = air.instData(comptime_tag, inst);
97 try w.writeAll("(");
98 try s.dumpRef(w, bin.operand);
99 try w.writeAll(")");
100 },
101 .field => {
102 const field: Air.Field = air.instData(.field, inst);
103 try w.writeAll("(");
104 try s.dumpRef(w, field.base);
105 try w.writeAll(", ");
106 try tty.setColor(w, .magenta);
107 try w.print("{s}", .{field.field.toSlice(s.ip)});
108 try tty.setColor(w, .reset);
109 try w.writeAll(")");
110 },
111 .call => {
112 const call: Air.Call = air.instData(.call, inst);
113 const call_data = call.call_data.get(air);
114 try w.writeAll("(");
115 try s.dumpRef(w, call.base);
116 try w.writeAll(", ");
117 try s.dumpRefList(w, call_data.args.slice(air), '[', ']');
118 try w.writeAll(")");
119 },
120 .@"if" => {
121 const if_inst: Air.If = air.instData(.@"if", inst);
122 const if_data: Air.If.Data = if_inst.data.get(air);
123 try w.writeAll("(");
124 try s.dumpRef(w, if_inst.cond);
125 try w.writeAll(", {");
126 s.indent += 1;
127 try s.newline(w);
128 try w.writeAll("yes: ");
129 try s.dumpRefRecurse(w, if_data.yes);
130 try s.newline(w);
131 try w.writeAll("no: ");
132 try s.dumpRefRecurse(w, if_data.no);
133 s.indent -= 1;
134 try s.newline(w);
135 try w.writeAll("}");
136 },
137 }
138 }
139
140 fn dumpRef(s: *State, w: anytype, ref: Inst.Ref) !void {
141 switch (ref.unwrap()) {
142 .inst => |i| try s.dumpInstRef(w, i),
143 .value => |v| try s.dumpValue(w, v),
144 }
145 }
146
147 fn dumpRefRecurse(s: *State, w: anytype, ref: Inst.Ref) !void {
148 switch (ref.unwrap()) {
149 .inst => |i| try s.dumpInst(w, i),
150 .value => |v| try s.dumpValue(w, v),
151 }
152 }
153
154 fn dumpInstRef(s: *State, w: anytype, inst: Inst.Index) @TypeOf(w).Error!void {
155 try s.tty.setColor(w, .red);
156 try w.print("#{d}", .{@intFromEnum(inst)});
157 try s.tty.setColor(w, .reset);
158 }
159
160 fn dumpValue(s: *State, w: anytype, val: Value) !void {
161 const tty = s.tty;
162 switch (val.toKey(s.ip)) {
163 .simple_type => |t| {
164 try tty.setColor(w, .blue);
165 try w.print("{s}", .{@tagName(t)});
166 try tty.setColor(w, .reset);
167 },
168 .simple_value => |t| {
169 try tty.setColor(w, .yellow);
170 try w.print("{s}", .{@tagName(t)});
171 try tty.setColor(w, .reset);
172 },
173 .type_function => |ty| {
174 try tty.setColor(w, .blue);
175 try w.writeAll("fn(");
176 try tty.setColor(w, .reset);
177 for (ty.param_types, 0..) |param, i| {
178 if (i != 0) {
179 try tty.setColor(w, .blue);
180 try w.writeAll(",");
181 try tty.setColor(w, .reset);
182 }
183 try s.dumpValue(w, Value.fromInterned(param));
184 }
185 try tty.setColor(w, .blue);
186 try w.writeAll(") -> ");
187 try tty.setColor(w, .reset);
188 try s.dumpValue(w, Value.fromInterned(ty.return_type));
189 },
190 inline .float, .int32 => |data, tag| {
191 try tty.setColor(w, .bright_black);
192 try w.writeAll("<");
193 try tty.setColor(w, .blue);
194 try w.writeAll(@tagName(tag));
195 try tty.setColor(w, .reset);
196 try tty.setColor(w, .bright_black);
197 try w.writeAll(", ");
198 try tty.setColor(w, .yellow);
199 try w.print("{d}", .{data});
200 try tty.setColor(w, .bright_black);
201 try w.writeAll(">");
202 try tty.setColor(w, .reset);
203 },
204 .string => |data| {
205 try w.writeAll("<");
206 try tty.setColor(w, .blue);
207 try w.writeAll("string");
208 try tty.setColor(w, .reset);
209 try w.writeAll(", ");
210 try tty.setColor(w, .green);
211 try w.writeAll("\"");
212 try w.writeAll(data);
213 try w.writeAll("\"");
214 try tty.setColor(w, .reset);
215 try w.writeAll(">");
216 },
217 .func => |func| {
218 try tty.setColor(w, .bright_black);
219 try w.writeAll("<");
220 try s.dumpValue(w, Value.fromInterned(func.ty));
221 try tty.setColor(w, .bright_black);
222 try w.writeAll(", ");
223 const decl_name = s.ip.declPtrConst(func.owner_decl).name.toSlice(s.ip);
224 try tty.setColor(w, .bright_green);
225 try w.print("decl '{s}'", .{decl_name});
226 try tty.setColor(w, .bright_black);
227 try w.writeAll(">");
228 try tty.setColor(w, .reset);
229 },
230 }
231 }
232
233 fn dumpInstList(state: *State, writer: anytype, list: []const Inst.Index, l: u8, r: u8) !void {
234 try writer.print(" {c}", .{l});
235 if (list.len > 0) {
236 state.indent += 1;
237
238 for (list) |inst| {
239 try state.newline(writer);
240 try state.dumpInst(writer, inst);
241 }
242
243 state.indent -= 1;
244 try state.newline(writer);
245 }
246 try writer.print("{c}", .{r});
247 }
248
249 fn dumpRefList(state: *State, writer: anytype, list: []const Inst.Ref, l: u8, r: u8) !void {
250 try writer.print(" {c}", .{l});
251 if (list.len > 0) {
252 state.indent += 1;
253
254 for (list) |inst| {
255 try state.newline(writer);
256 try state.dumpRef(writer, inst);
257 }
258
259 state.indent -= 1;
260 try state.newline(writer);
261 }
262 try writer.print("{c}", .{r});
263 }
264};
265
266pub fn format(
267 dump: @This(),
268 comptime unused_fmt: []const u8,
269 _: std.fmt.FormatOptions,
270 writer: anytype,
271) !void {
272 comptime assert(unused_fmt.len == 0);
273
274 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
275 defer arena.deinit();
276
277 const air = dump.air;
278 var state = State{
279 .air = &air,
280 .ip = dump.ip,
281 .tty = detectTTYFromWriter(writer),
282 .arena = arena.allocator(),
283 };
284 try state.dumpInst(writer, .main_block);
285 try writer.writeAll("\n");
286 assert(state.indent == 0);
287}
288
289const std = @import("std");
290const assert = std.debug.assert;
291
292const Air = @import("Air.zig");
293const Inst = Air.Inst;
294
295const Value = @import("Value.zig");
296
297const InternPool = @import("InternPool.zig");
298
299const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter;
src/compiler/Air.zig created+292
......@@ -0,0 +1,292 @@
1insts: Inst.List.Slice,
2extra: ExtraStore = .{},
3
4pub const ExtraStore = dod.ExtraStore(.air, .@"const");
5pub const EncodedData = dod.EncodedData(Inst.Tag, u64);
6pub const ExtraIndex = ExtraStore.Index;
7
8pub const Inst = struct {
9 tag: Tag,
10 data: EncodedData,
11 flags: Flags,
12
13 pub const List = std.MultiArrayList(Inst);
14
15 pub const Tag = enum {
16 block,
17 js_global_ref,
18 call,
19 @"break",
20 @"if",
21 arg,
22 field,
23
24 // bin
25 add,
26 sub,
27 mul,
28 div,
29
30 eq,
31
32 lt,
33 gt,
34
35 // un
36 negate,
37
38 pub fn Data(comptime tag: Tag) type {
39 return switch (tag) {
40 .block => IndexList,
41 .js_global_ref => InternPool.String.WithLen,
42 .@"break" => Break,
43 .@"if" => If,
44 .call => Call,
45 .arg => Arg,
46 .add,
47 .sub,
48 .mul,
49 .div,
50 .lt,
51 .gt,
52 .eq,
53 => Bin,
54 .negate,
55 => Un,
56 .field,
57 => Field,
58 };
59 }
60 };
61
62 /// reference to either a value or another instruction in this air
63 pub const Ref = packed struct(u32) {
64 kind: Kind,
65 data: u31,
66
67 pub const Kind = enum(u1) { value, inst };
68
69 pub const Decoded = union(enum) {
70 value: Value,
71 inst: Index,
72 };
73
74 pub fn unwrap(ref: Ref) Decoded {
75 return switch (ref.kind) {
76 .value => .{ .value = Value.fromInterned(@enumFromInt(ref.data)) },
77 .inst => .{ .inst = @enumFromInt(ref.data) },
78 };
79 }
80
81 pub fn toOptional(ref: ?Ref) Optional {
82 return if (ref) |r| @enumFromInt(@as(u32, @bitCast(r))) else .none;
83 }
84
85 pub const Optional = enum(u32) {
86 none = std.math.maxInt(u32),
87 _,
88
89 pub fn unwrap(opt: Optional) ?Ref {
90 return if (opt != .none) @bitCast(@intFromEnum(opt)) else null;
91 }
92 };
93 };
94
95 /// reference to another instruction in this air
96 /// note that these cannot point to code in other functions, such makes no sense
97 pub const Index = enum(u32) {
98 main_block,
99 _,
100
101 pub fn toRef(index: Index) Ref {
102 return .{
103 .kind = .inst,
104 .data = @intCast(@intFromEnum(index)),
105 };
106 }
107
108 pub fn toOptional(index: Index) Optional {
109 return @enumFromInt(@intFromEnum(index));
110 }
111
112 pub const Optional = enum(u32) {
113 none = std.math.maxInt(u32),
114 _,
115
116 pub fn unwrap(oi: Optional) ?Index {
117 return if (oi != .none) @enumFromInt(@intFromEnum(oi)) else null;
118 }
119 };
120 };
121};
122
123pub const Flags = packed struct {
124 uses: Uses = .none,
125
126 pub const Uses = enum(u2) {
127 none,
128 single,
129 multi,
130
131 pub fn increment(uses: Uses) Uses {
132 return switch (uses) {
133 .none => .single,
134 .single, .multi => .multi,
135 };
136 }
137 };
138};
139
140pub const IndexList = struct {
141 start: u32,
142 len: u32,
143
144 pub fn slice(self: IndexList, air: *const Air) []const Inst.Index {
145 return @ptrCast(air.extra.list.items[self.start..][0..self.len]);
146 }
147};
148
149pub const RefList = struct {
150 start: u32,
151 len: u32,
152
153 pub fn slice(self: RefList, air: *const Air) []const Inst.Ref {
154 return @ptrCast(air.extra.list.items[self.start..][0..self.len]);
155 }
156};
157
158pub const Break = struct {
159 block_ref: Inst.Index,
160 value: Inst.Ref,
161};
162
163pub const Bin = struct {
164 lhs: Inst.Ref,
165 rhs: Inst.Ref,
166};
167
168pub const Un = struct {
169 operand: Inst.Ref,
170 unused: Unused = .zero,
171};
172
173pub const Field = struct {
174 base: Inst.Ref,
175 field: InternPool.NullTerminatedString,
176};
177
178pub const Call = struct {
179 base: Inst.Ref,
180 call_data: ExtraIndex(Data),
181
182 pub const Data = struct {
183 args: RefList,
184 };
185};
186
187pub const Arg = struct {
188 type: Value,
189 arg_number: u32,
190};
191
192pub const If = struct {
193 cond: Inst.Ref,
194 data: ExtraIndex(Data),
195
196 pub const Data = struct {
197 yes: Inst.Ref,
198 no: Inst.Ref,
199 union_ty: Value,
200 };
201};
202
203pub fn deinit(air: *Air, gpa: std.mem.Allocator) void {
204 defer air.* = undefined;
205 air.insts.deinit(gpa);
206 air.extra.deinit(gpa);
207}
208
209pub fn instTag(air: *const Air, index: Inst.Index) Inst.Tag {
210 return air.insts.items(.tag)[@intFromEnum(index)];
211}
212
213pub fn instFlags(air: *const Air, index: Inst.Index) Flags {
214 return air.insts.items(.flags)[@intFromEnum(index)];
215}
216
217pub fn instEncodedData(air: *const Air, index: Inst.Index) EncodedData {
218 return air.insts.items(.data)[@intFromEnum(index)];
219}
220
221pub fn instData(air: *const Air, comptime tag: Inst.Tag, index: Inst.Index) tag.Data() {
222 return air.instEncodedData(index).decode(tag, air.instTag(index));
223}
224
225pub fn instPayload(air: *const Air, comptime tag: Inst.Tag, index: Inst.Index) tag.Data().T {
226 return air.instData(tag, index).extra_index.get(air);
227}
228
229pub fn typeOf(air: *const Air, ref: Inst.Ref, ip: *InternPool) InternPool.Index {
230 return switch (ref.unwrap()) {
231 .value => |v| v.type(ip).toIntern(),
232 .inst => |i| air.typeOfIndex(i, ip),
233 };
234}
235
236pub fn typeOfIndex(air: *const Air, index: Inst.Index, ip: *InternPool) InternPool.Index {
237 return switch (air.instTag(index)) {
238 .js_global_ref => .opaque_type,
239
240 .block => {
241 const items = air.instData(.block, index).slice(air);
242 return air.typeOfIndex(items[items.len - 1], ip);
243 },
244
245 inline .add, .sub, .mul, .div => |t| air.typeOf(air.instData(t, index).lhs, ip),
246
247 .lt, .gt, .eq => .bool_type,
248
249 .arg => air.instData(.arg, index).type.toIntern(),
250
251 .negate => air.typeOf(air.instData(.negate, index).operand, ip),
252
253 .@"break" => .noreturn_type,
254
255 // todo:
256 .field => .opaque_type,
257
258 .call => {
259 const base_type = air.typeOf(air.instData(.call, index).base, ip);
260 if (base_type == .opaque_type) return .opaque_type;
261 return base_type.toKey(ip).type_function.return_type;
262 },
263
264 .@"if" => air.instData(.@"if", index).data.get(air).union_ty.toIntern(),
265 };
266}
267
268pub fn paramCount(air: Air) u32 {
269 for (air.insts.items(.tag)[1..], 0..) |tag, i| {
270 if (tag != .arg) return @intCast(i);
271 }
272 return @intCast(air.insts.len);
273}
274
275pub fn dumpToWriter(air: Air, ip: *InternPool, writer: anytype) !void {
276 try writer.print("{}", .{Dump{ .air = air, .ip = ip }});
277}
278
279pub const Dump = @import("Air.Dump.zig");
280
281const Air = @This();
282const std = @import("std");
283const mem = std.mem;
284const assert = std.debug.assert;
285
286const Teg = @import("Teg.zig");
287const InternPool = @import("InternPool.zig");
288const dod = @import("dod.zig");
289
290const Unused = enum(u32) { zero = 0 };
291
292const Value = @import("Value.zig");
src/compiler/Ast.Dump.zig created+300
......@@ -0,0 +1,300 @@
1//! A flat array is not very useful for debugging, so this structure implements
2//! a debug formatter for Ast.
3ast: Ast,
4
5const State = struct {
6 ast: *const Ast,
7 indent: usize = 0,
8 node_tags: []const Node.Tag,
9 node_data: []const Node.Data,
10 node_tokens: []const TokenIndex,
11 tty: std.io.tty.Config,
12
13 fn newline(state: State, writer: anytype) !void {
14 try writer.writeByte('\n');
15 if (state.indent > 0)
16 try writer.writeByteNTimes(' ', state.indent * 2);
17 }
18};
19
20pub fn format(
21 dump: Dump,
22 comptime unused_fmt: []const u8,
23 _: std.fmt.FormatOptions,
24 writer: anytype,
25) !void {
26 comptime assert(unused_fmt.len == 0);
27 const ast = dump.ast;
28 var state = State{
29 .ast = &ast,
30 .node_tags = ast.nodes.items(.tag),
31 .node_data = ast.nodes.items(.data),
32 .node_tokens = ast.nodes.items(.main_token),
33 .tty = detectTTYFromWriter(writer),
34 };
35 try dumpNode(writer, &state, 0);
36 try writer.writeAll("\n");
37 assert(state.indent == 0);
38}
39
40fn dumpNode(writer: anytype, state: *State, node: NodeIndex) @TypeOf(writer).Error!void {
41 const tag = state.node_tags[node];
42 const data = state.node_data[node];
43 const data_tag = Node.data_tags[@intFromEnum(tag)];
44 const main_token = state.node_tokens[node];
45 const tty = state.tty;
46
47 try state.tty.setColor(writer, .bright_black);
48 try writer.print("{d}", .{node});
49 try state.tty.setColor(writer, .reset);
50 try writer.writeAll(" ");
51
52 try tty.setColor(writer, .cyan);
53 try writer.print("{s}", .{@tagName(tag)});
54 try tty.setColor(writer, .reset);
55
56 switch (tag) {
57 .builtin_call => {
58 try tty.setColor(writer, .red);
59 try writer.print(" {s}", .{state.ast.tokenSlice(main_token)});
60 try tty.setColor(writer, .reset);
61 },
62 else => {},
63 }
64
65 switch (data_tag) {
66 .none => {
67 switch (tag) {
68 .id => try tty.setColor(writer, .red),
69 .string => try tty.setColor(writer, .green),
70 .number_int, .number_float => try tty.setColor(writer, .yellow),
71 else => {},
72 }
73 switch (tag) {
74 .id,
75 .string,
76 .number_int,
77 .number_float,
78 => try writer.print(" {s}", .{state.ast.tokenSlice(main_token)}),
79
80 else => unreachable,
81 }
82 try tty.setColor(writer, .reset);
83 },
84 .fn_decl => {
85 try writer.writeAll(" ");
86 const fn_decl = state.ast.fullFnDecl(node);
87 if (fn_decl.public) {
88 try writer.writeAll("[pub] ");
89 }
90 try tty.setColor(writer, .magenta);
91 try writer.writeAll(fn_decl.id);
92 try tty.setColor(writer, .reset);
93 try writer.writeAll(" {");
94 {
95 state.indent += 1;
96 defer state.indent -= 1;
97 try state.newline(writer);
98 try writer.writeAll("params:");
99 try dumpNodeList(writer, state, fn_decl.params, '[', ']');
100 if (fn_decl.return_type) |return_type| {
101 try state.newline(writer);
102 try writer.writeAll("return_type: ");
103 try dumpNode(writer, state, return_type);
104 }
105 try state.newline(writer);
106 try writer.writeAll("body: ");
107 try dumpNode(writer, state, fn_decl.body);
108 }
109 try state.newline(writer);
110 try writer.writeAll("}");
111 },
112 .list => {
113 const list = data.list.slice(state.ast);
114 switch (tag) {
115 .root,
116 .block_void,
117 .block_implicit_return,
118 => try dumpNodeList(writer, state, list, '{', '}'),
119 .builtin_call,
120 => try dumpNodeList(writer, state, list, '(', ')'),
121 else => unreachable,
122 }
123 },
124 .var_decl => {
125 try writer.writeAll(" ");
126 const var_decl = state.ast.fullVarDecl(node);
127 if (var_decl.public) {
128 try writer.writeAll("[pub] ");
129 }
130 try writer.writeAll(switch (var_decl.is_const) {
131 true => "constant",
132 false => "mutable",
133 });
134 try tty.setColor(writer, .magenta);
135 try writer.print(" {s}", .{var_decl.id});
136 try tty.setColor(writer, .reset);
137 var indent = false;
138 if (var_decl.type) |type_node| {
139 state.indent += 1;
140 indent = true;
141 try state.newline(writer);
142 try writer.writeAll(": ");
143 try dumpNode(writer, state, type_node);
144 try writer.writeAll(", ");
145 }
146 try writer.writeAll(" = ");
147 try dumpNode(writer, state, var_decl.init);
148 if (indent) {
149 state.indent -= 1;
150 }
151 },
152 .param_decl => {
153 try writer.writeAll(" ");
154 const param_decl = state.ast.fullParamDecl(node);
155 try tty.setColor(writer, .magenta);
156 try writer.writeAll(param_decl.id);
157 try tty.setColor(writer, .reset);
158 try writer.writeAll(": ");
159 try dumpNode(writer, state, param_decl.type);
160 },
161 .bin => {
162 try writer.writeAll(" (");
163 {
164 state.indent += 1;
165 defer state.indent -= 1;
166 try state.newline(writer);
167 try dumpNode(writer, state, data.bin.lhs);
168 try state.newline(writer);
169 try dumpNode(writer, state, data.bin.rhs);
170 }
171 try state.newline(writer);
172 try writer.writeAll(")");
173 },
174 .un => {
175 try writer.writeAll(" (");
176 {
177 state.indent += 1;
178 defer state.indent -= 1;
179 try state.newline(writer);
180 try dumpNode(writer, state, data.un.expr);
181 }
182 try state.newline(writer);
183 try writer.writeAll(")");
184 },
185 .label => {
186 const label_text = state.ast.tokenSlice(main_token);
187 try writer.print(" {s}:", .{label_text});
188
189 try writer.writeAll(" (");
190 {
191 state.indent += 1;
192 defer state.indent -= 1;
193 try state.newline(writer);
194 try dumpNode(writer, state, data.label.expr);
195 }
196 try state.newline(writer);
197 try writer.writeAll(")");
198 },
199 .field => {
200 try writer.writeAll(" (");
201 {
202 state.indent += 1;
203 defer state.indent -= 1;
204 try state.newline(writer);
205 try dumpNode(writer, state, data.field.base);
206 }
207 try state.newline(writer);
208 try writer.writeAll(")");
209 },
210 .@"return" => {
211 if (data.@"return".expr.unwrap()) |expr| {
212 try writer.writeAll(" (");
213 try dumpNode(writer, state, expr);
214 try writer.writeAll(")");
215 }
216 },
217 .@"break" => {
218 const label = data.@"break".label;
219 const value = data.@"break".value;
220 if (label != .none or value != .none) {
221 // try writer.writeAll("(");
222 // try writer.writeAll(")");
223 if (label.unwrap()) |l| {
224 const label_text = state.ast.tokenSlice(l);
225 try writer.print(" :{s}", .{label_text});
226 }
227 if (value.unwrap()) |expr| {
228 try writer.writeAll(" (");
229 try dumpNode(writer, state, expr);
230 try writer.writeAll(")");
231 }
232 }
233 },
234 .call => {
235 const base = data.call.base;
236 const extra = state.ast.extraData(data.call.extra, Call);
237 const arg_list = extra.args.slice(state.ast);
238
239 try writer.writeAll("(");
240 try writer.writeAll("base: ");
241 try dumpNode(writer, state, base);
242 try writer.writeAll(", ");
243 try writer.writeAll("arg:");
244 try dumpNodeList(writer, state, arg_list, '[', ']');
245 try writer.writeAll(")");
246 },
247 .@"if" => {
248 const branch = data.@"if";
249 const if_data = state.ast.extraData(branch.extra, Ast.If);
250
251 try writer.writeAll(" (");
252 try dumpNode(writer, state, branch.cond);
253 try writer.writeAll(") {");
254 {
255 state.indent += 1;
256 defer state.indent -= 1;
257 try state.newline(writer);
258 try writer.writeAll("yes: ");
259 try dumpNode(writer, state, if_data.yes);
260 try state.newline(writer);
261 if (if_data.no.unwrap()) |no| {
262 try writer.writeAll("no: ");
263 try dumpNode(writer, state, no);
264 } else {
265 try writer.writeAll("no: [none]");
266 }
267 }
268 try state.newline(writer);
269 try writer.writeAll("}");
270 },
271 }
272}
273
274fn dumpNodeList(writer: anytype, state: *State, list: []const NodeIndex, l: u8, r: u8) !void {
275 try writer.print(" {c}", .{l});
276 {
277 state.indent += 1;
278 defer state.indent -= 1;
279
280 for (list) |node| {
281 try state.newline(writer);
282 try dumpNode(writer, state, node);
283 }
284 }
285 try state.newline(writer);
286 try writer.print("{c}", .{r});
287}
288
289const Dump = @This();
290
291const std = @import("std");
292const assert = std.debug.assert;
293
294const Ast = @import("Ast.zig");
295const NodeIndex = Ast.NodeIndex;
296const Node = Ast.Node;
297const TokenIndex = Ast.TokenIndex;
298const Call = Ast.Call;
299
300const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter;
src/compiler/Ast.zig created+701
......@@ -0,0 +1,701 @@
1//! The Abstract Syntax Tree preserves most details of the source code,
2//! as it is used for formatting. Before semantic analysis, this
3//! structure is normalized into `Teg`, which removes constructs like
4//! grouping, unneded let bindings, and other transforms.
5//!
6//! `ast.nodes` contains nodes stored as three parts:
7//! - tag: the type of the node
8//! - main_token: index to it's primary token, depends on the tag
9//! - data: two u32 values, depends on the tag
10//!
11//! Many nodes encode all their metadata with just `node.data`, using
12//! facts about the language grammar to be able to infer data without
13//! storing it.
14//!
15//! For example, `var_decl` has the following encoding:
16//! - `main_token` is either a `kw_var` or `kw_let`
17//! - the token after `main_token` is always the identifier
18//! - `data.var_decl` contains `NodeIndex`es pointing at the type
19//! and the initializer, which are both expressions.
20//! - If the keyword before `main_token` is `kw_pub` it is a public variable.
21//!
22//! To make decoding these easier, many nodes contain a "full" variant,
23//! such as `full.VarDecl`, which can be aquired with `ast.fullVarDecl(id)`.
24//!
25//! For nodes that must store more than 64 bits of payload data, an "extra"
26//! struct is stored out-of-line in the `extra` field. These can be read
27//! using `ast.extraData(extra_index, T)`. Again, it is recommended to use
28//! the `full` structures with their convenience functions.
29//!
30//! Documentation comments are not stored anywhere, but you can find them
31//! by iterating backwards from a declaration and seeing if a `.doc_comment`
32//! is present. This is provided by `ast.declDocComment(id)`.
33//!
34//! The `Ast` owns token and node data, but not the source code.
35
36/// Externally owned source code.
37utf8_source: []const u8,
38
39tokens: Token.List.Slice,
40nodes: Node.List.Slice,
41extra: []const u32,
42
43pub fn parse(gpa: std.mem.Allocator, source: []const u8) !Ast {
44 const Parse = @import("Parse.zig");
45
46 var todo: ByteOffset = 0;
47 var tokens = try Tokenizer.tokenize(source, gpa, &todo);
48 errdefer tokens.deinit(gpa);
49
50 var parser = Parse.init(gpa, source, tokens);
51 errdefer parser.deinit();
52
53 try parser.nodes.ensureUnusedCapacity(gpa, @divFloor(tokens.len, 4));
54
55 try parser.parseRoot();
56
57 return try parser.finalize(tokens);
58}
59
60pub const Location = struct {
61 line: usize,
62 column: usize,
63 line_start: usize,
64 line_end: usize,
65};
66
67pub const Span = struct {
68 start: u32,
69 end: u32,
70 main: u32,
71};
72
73// This can be removed when the following issue is implemented:
74// See: https://github.com/ziglang/zig/issues/104
75// See: https://github.com/ziglang/zig/issues/3806
76pub const OptionalNodeIndex = enum(u32) {
77 none = 0,
78 _,
79
80 pub fn init(idx: ?NodeIndex) OptionalNodeIndex {
81 assert(idx != 0);
82 return @enumFromInt(idx orelse return .none);
83 }
84
85 pub fn unwrap(idx: OptionalNodeIndex) ?NodeIndex {
86 return switch (idx) {
87 .none => null,
88 _ => @intFromEnum(idx),
89 };
90 }
91};
92
93pub const OptionalTokenIndex = enum(u32) {
94 none = 0,
95 _,
96
97 pub fn init(idx: ?TokenIndex) OptionalTokenIndex {
98 assert(idx != 0);
99 return @enumFromInt(idx orelse return .none);
100 }
101
102 pub fn unwrap(idx: OptionalTokenIndex) ?TokenIndex {
103 return switch (idx) {
104 .none => null,
105 _ => @intFromEnum(idx),
106 };
107 }
108};
109
110pub const OptionalExtraIndex = enum(u32) {
111 none = std.math.maxInt(u32),
112 _,
113
114 pub fn init(idx: ?ExtraIndex) OptionalExtraIndex {
115 assert(idx != std.math.maxInt(u32));
116 return @enumFromInt(idx orelse return .none);
117 }
118
119 pub fn unwrap(idx: OptionalExtraIndex) ?ExtraIndex {
120 return switch (idx) {
121 .none => null,
122 _ => @intFromEnum(idx),
123 };
124 }
125};
126
127pub const Node = struct {
128 tag: Tag,
129 main_token: TokenIndex,
130 data: Data,
131
132 pub const List = std.MultiArrayList(Node);
133
134 pub const Tag = enum {
135 /// Read with `ast.rootNodes()`
136 /// data is .list (containing stmt slice)
137 /// TODO: rename to `module`
138 root,
139 /// main_token is the {
140 /// data is .list (containing stmt slice)
141 ///
142 /// This guarantees that the block either empty, or the last statement
143 /// ends with a semicolon.
144 block_void,
145 /// main_token is the {
146 /// data is .list (containing stmt slice)
147 ///
148 /// This guarantees that the block is not empty and that the last
149 /// statement is an expression statement without a trailing semicolon.
150 block_implicit_return,
151 /// Variable or constant declaration
152 /// `var x: type = init;`
153 /// `let y = x;`
154 /// main_token is the keyword
155 /// data is .var_decl
156 var_decl,
157 /// Function declaration
158 /// `fn foo(x: i32) -> i32 { rhs }`
159 /// Params and return type are stored in extra data
160 fn_decl,
161 /// 'name: type' declaration
162 /// Main token is the identifier
163 param_decl,
164 /// An identifier. main_token is the identifier.
165 id,
166 /// Main token is the data. May not fit in i32 range,
167 /// but will contain no dots or e's.
168 number_int,
169 /// Main token is data. Will contain '.' and/or 'e'.
170 number_float,
171 /// A string literal
172 string,
173 /// A multi-line string literal
174 /// data is .token_list of the tokens, which is guaranteed to be contiguous
175 multiline_string,
176 /// Call to a function
177 /// `foo(x, y);`
178 /// main_token is the `(`
179 /// data is .call
180 call,
181 /// Call to a builtin function
182 /// `@raw("hello");`
183 /// data is .list (containing arg slice)
184 builtin_call,
185 /// `if cond { ... } else ...`
186 /// main_token is the `if`
187 /// data is .@"if"
188 @"if",
189 /// `if let id = ... { ... } else ...`
190 /// main_token is the `if`
191 /// data is .@"if"
192 // if_let_nonvoid,
193 /// `id: ...`
194 /// data is un
195 label,
196 /// `expr.field`
197 /// main_token is the `.`
198 /// data is .field
199 field,
200 /// `lhs[rhs]`
201 /// main_token is the `[`
202 /// data is .bin
203 index,
204 /// `return expr;`
205 /// main_token is the `return`
206 /// data is .@"return"
207 @"return",
208 /// `break :brk expr;`
209 /// main_token is the `break`
210 /// data is .@"break"
211 @"break",
212 /// `(expr)`
213 /// main_token is the `(`
214 /// data is .un
215 group,
216 /// `lhs + rhs`. main_token is the operator.
217 add,
218 /// `lhs - rhs`. main_token is the operator.
219 sub,
220 /// `lhs * rhs`. main_token is the operator.
221 mul,
222 /// `lhs / rhs`. main_token is the operator.
223 div,
224 /// `lhs % rhs`. main_token is the operator.
225 rem,
226 /// `lhs ** rhs`. main_token is the operator.
227 pow,
228 /// `lhs && rhs`. main_token is the operator.
229 @"and",
230 /// `lhs || rhs`. main_token is the operator.
231 @"or",
232 /// `lhs & rhs`. main_token is the operator.
233 bit_and,
234 /// `lhs | rhs`. main_token is the operator.
235 bit_or,
236 /// `lhs ^ rhs`. main_token is the operator.
237 bit_xor,
238 /// `lhs << rhs`. main_token is the operator.
239 shl,
240 /// `lhs >> rhs`. main_token is the operator.
241 shr,
242 /// `lhs == rhs`. main_token is the operator.
243 eq,
244 /// `lhs != rhs`. main_token is the operator.
245 ne,
246 /// `lhs < rhs`. main_token is the operator.
247 lt,
248 /// `lhs <= rhs`. main_token is the operator.
249 le,
250 /// `lhs > rhs`. main_token is the operator.
251 gt,
252 /// `lhs >= rhs`. main_token is the operator.
253 ge,
254 /// `lhs = rhs`. main_token is the operator.
255 assign,
256 /// `lhs += rhs`. main_token is the operator.
257 add_assign,
258 /// `lhs -= rhs`. main_token is the operator.
259 sub_assign,
260 /// `lhs *= rhs`. main_token is the operator.
261 mul_assign,
262 /// `lhs /= rhs`. main_token is the operator.
263 div_assign,
264 /// `lhs %= rhs`. main_token is the operator.
265 rem_assign,
266 /// `lhs **= rhs`. main_token is the operator.
267 pow_assign,
268 /// `lhs &&= rhs`. main_token is the operator.
269 and_assign,
270 /// `lhs ||= rhs`. main_token is the operator.
271 or_assign,
272 /// `lhs &= rhs`. main_token is the operator.
273 bit_and_assign,
274 /// `lhs |= rhs`. main_token is the operator.
275 bit_or_assign,
276 /// `lhs ^= rhs`. main_token is the operator.
277 bit_xor_assign,
278 /// `lhs <<= rhs`. main_token is the operator.
279 shl_assign,
280 /// `lhs >>= rhs`. main_token is the operator.
281 shr_assign,
282 /// `!expr`. main_token is the operator.
283 bool_not,
284 /// `-expr`. main_token is the operator.
285 negate,
286 /// `~expr`. main_token is the operator.
287 bit_not,
288 /// `comptime expr`. main_token is the operator.
289 @"comptime",
290 /// `pure expr`. main_token is the operator.
291 // pure,
292 /// `assert expr`. main_token is the operator.
293 // assert,
294
295 pub fn isAssignBin(op: Tag) bool {
296 return switch (op) {
297 .assign,
298 .add_assign,
299 .sub_assign,
300 .mul_assign,
301 .div_assign,
302 .rem_assign,
303 .pow_assign,
304 .and_assign,
305 .or_assign,
306 .bit_and_assign,
307 .bit_or_assign,
308 .bit_xor_assign,
309 .shl_assign,
310 .shr_assign,
311 => true,
312 .add,
313 .sub,
314 .mul,
315 .div,
316 .rem,
317 .pow,
318 .@"and",
319 .@"or",
320 .bit_and,
321 .bit_or,
322 .bit_xor,
323 .shl,
324 .shr,
325 .eq,
326 .ne,
327 .lt,
328 .le,
329 .gt,
330 .ge,
331 => false,
332 else => unreachable, // not a binary operator
333 };
334 }
335
336 pub fn isBasic(op: Tag) bool {
337 return switch (op) {
338 .id,
339 .string,
340 .number_int,
341 .number_float,
342 .builtin_call,
343 => true,
344 else => false,
345 };
346 }
347 };
348
349 pub const data_tags = std.enums.directEnumArray(Tag, Data.FieldEnum, 0, .{
350 .root = .list,
351 .block_void = .list,
352 .block_implicit_return = .list,
353 .var_decl = .var_decl,
354 .fn_decl = .fn_decl,
355 .param_decl = .param_decl,
356 .id = .none,
357 .number_int = .none,
358 .number_float = .none,
359 .string = .none,
360 .multiline_string = .list,
361 .call = .call,
362 .builtin_call = .list,
363 .@"if" = .@"if",
364 .field = .field,
365 .index = .bin,
366 .@"return" = .@"return",
367 .@"break" = .@"break",
368 .group = .un,
369 .add = .bin,
370 .sub = .bin,
371 .mul = .bin,
372 .div = .bin,
373 .rem = .bin,
374 .pow = .bin,
375 .@"and" = .bin,
376 .@"or" = .bin,
377 .bit_and = .bin,
378 .bit_or = .bin,
379 .bit_xor = .bin,
380 .shl = .bin,
381 .shr = .bin,
382 .eq = .bin,
383 .ne = .bin,
384 .lt = .bin,
385 .le = .bin,
386 .gt = .bin,
387 .ge = .bin,
388 .assign = .bin,
389 .add_assign = .bin,
390 .sub_assign = .bin,
391 .mul_assign = .bin,
392 .div_assign = .bin,
393 .rem_assign = .bin,
394 .pow_assign = .bin,
395 .and_assign = .bin,
396 .or_assign = .bin,
397 .bit_and_assign = .bin,
398 .bit_or_assign = .bin,
399 .bit_xor_assign = .bin,
400 .shl_assign = .bin,
401 .shr_assign = .bin,
402 .bool_not = .un,
403 .negate = .un,
404 .bit_not = .un,
405 .@"comptime" = .un,
406 .label = .label,
407 });
408
409 pub const Data = union {
410 none: struct { Unused = .zero, Unused = .zero },
411 fn_decl: struct {
412 // A `FnDecl`
413 extra: ExtraIndex,
414 body: NodeIndex,
415 },
416 /// either is a node list or a token list, depending on the node's tag
417 list: SubList,
418 var_decl: struct {
419 type: OptionalNodeIndex,
420 init: NodeIndex,
421 },
422 param_decl: struct {
423 type: NodeIndex,
424 unused: Unused = .zero,
425 },
426 bin: struct {
427 lhs: NodeIndex,
428 rhs: NodeIndex,
429 },
430 field: struct {
431 base: NodeIndex,
432 field_token: TokenIndex,
433 },
434 un: struct {
435 expr: NodeIndex,
436 unused: Unused = .zero,
437 },
438 label: struct {
439 expr: NodeIndex,
440 unused: Unused = .zero,
441 },
442 @"return": struct {
443 expr: OptionalNodeIndex,
444 unused: Unused = .zero,
445 },
446 @"break": struct {
447 label: OptionalTokenIndex,
448 value: OptionalNodeIndex,
449 },
450 call: struct {
451 base: NodeIndex,
452 /// A `Call`
453 extra: ExtraIndex,
454 },
455 @"if": struct {
456 cond: NodeIndex,
457 /// A `If`
458 extra: ExtraIndex,
459 },
460
461 const FieldEnum = std.meta.FieldEnum(Data);
462 };
463};
464
465pub const SubList = struct {
466 /// Either NodeIndex or TokenIndex, depending on where this came from.
467 start: u32,
468 len: u32,
469
470 pub fn slice(self: SubList, ast: *const Ast) []const u32 {
471 return ast.extra[self.start..][0..self.len];
472 }
473};
474
475pub const FnDecl = struct {
476 params: SubList,
477 return_type: OptionalNodeIndex,
478};
479
480pub const Call = struct {
481 args: SubList,
482};
483
484pub const If = struct {
485 /// either .block_void or .block_implicit_return
486 yes: NodeIndex,
487 /// if it exists, it is either
488 /// .block_void, .block_implicit_return, or .@"if"
489 no: OptionalNodeIndex,
490};
491
492pub const full = struct {
493 pub const FnDecl = struct {
494 start_token: TokenIndex,
495 public: bool,
496 id: []const u8,
497 params: []const NodeIndex,
498 return_type: ?NodeIndex,
499 body: NodeIndex,
500 };
501
502 pub const VarDecl = struct {
503 start_token: TokenIndex,
504 public: bool,
505 is_const: bool,
506 id: []const u8,
507 type: ?NodeIndex,
508 init: NodeIndex,
509 };
510
511 pub const ParamDecl = struct {
512 start_token: TokenIndex,
513 id: []const u8,
514 type: NodeIndex,
515 };
516};
517
518pub fn deinit(ast: *Ast, gpa: std.mem.Allocator) void {
519 ast.tokens.deinit(gpa);
520 ast.nodes.deinit(gpa);
521 gpa.free(ast.extra);
522 ast.* = undefined;
523}
524
525pub fn nodeTag(ast: Ast, node: NodeIndex) Node.Tag {
526 return ast.nodes.items(.tag)[node];
527}
528
529pub fn nodeData(ast: Ast, node: NodeIndex) Node.Data {
530 return ast.nodes.items(.data)[node];
531}
532
533pub fn nodeToken(ast: Ast, node: NodeIndex) TokenIndex {
534 return ast.nodes.items(.main_token)[node];
535}
536
537pub fn fullFnDecl(ast: Ast, node: NodeIndex) full.FnDecl {
538 const tok_tags = ast.tokens.items(.tag);
539
540 const main_token = ast.nodes.items(.main_token)[node];
541 const public = tok_tags[main_token -| 1] == .kw_pub;
542 assert(tok_tags[main_token] == .kw_fn);
543 assert(tok_tags[main_token + 1] == .id);
544 const id = ast.tokenSlice(main_token + 1);
545
546 const data = ast.nodes.items(.data)[node].fn_decl;
547 const extra = ast.extraData(data.extra, FnDecl);
548
549 return .{
550 .start_token = if (public) main_token - 1 else main_token,
551 .public = public,
552 .id = id,
553 .body = data.body,
554 .params = extra.params.slice(&ast),
555 .return_type = extra.return_type.unwrap(),
556 };
557}
558
559pub fn fullVarDecl(ast: Ast, node: NodeIndex) full.VarDecl {
560 const tok_tags = ast.tokens.items(.tag);
561
562 const main_token = ast.nodes.items(.main_token)[node];
563 const public = tok_tags[main_token -| 1] == .kw_pub;
564 const is_const = switch (tok_tags[main_token]) {
565 .kw_let => true,
566 .kw_var => false,
567 else => |tag| {
568 if (std.debug.runtime_safety) {
569 std.debug.panic("Expected var decl, got {s}", .{@tagName(tag)});
570 }
571 unreachable;
572 },
573 };
574 assert(tok_tags[main_token + 1] == .id);
575 const id = ast.tokenSlice(main_token + 1);
576 const data = ast.nodes.items(.data)[node].var_decl;
577 return .{
578 .start_token = if (public) main_token - 1 else main_token,
579 .public = public,
580 .is_const = is_const,
581 .id = id,
582 .type = data.type.unwrap(),
583 .init = data.init,
584 };
585}
586
587pub fn fullParamDecl(ast: Ast, node: NodeIndex) full.ParamDecl {
588 const tok_tags = ast.tokens.items(.tag);
589
590 const main_token = ast.nodes.items(.main_token)[node];
591 assert(tok_tags[main_token] == .id);
592 const id = ast.tokenSlice(main_token);
593 const data = ast.nodes.items(.data)[node].param_decl;
594 return .{
595 .start_token = main_token,
596 .id = id,
597 .type = data.type,
598 };
599}
600
601pub fn nodeStatements(ast: *const Ast, node: NodeIndex) []const NodeIndex {
602 const data = ast.nodes.items(.data)[node].list;
603 return data.slice(ast);
604}
605
606pub fn tokenLocation(self: Ast, start_offset: ByteOffset, token_index: TokenIndex) Location {
607 var loc = Location{
608 .line = 0,
609 .column = 0,
610 .line_start = start_offset,
611 .line_end = self.source.len,
612 };
613 const token_start = self.tokens.items(.start)[token_index];
614
615 // Scan to by line until we go past the token start
616 while (std.mem.indexOfScalarPos(u8, self.source, loc.line_start, '\n')) |i| {
617 if (i >= token_start) {
618 break; // Went past
619 }
620 loc.line += 1;
621 loc.line_start = i + 1;
622 }
623
624 const offset = loc.line_start;
625 for (self.source[offset..], 0..) |c, i| {
626 if (i + offset == token_start) {
627 loc.line_end = i + offset;
628 while (loc.line_end < self.source.len and self.source[loc.line_end] != '\n') {
629 loc.line_end += 1;
630 }
631 return loc;
632 }
633 if (c == '\n') {
634 loc.line += 1;
635 loc.column = 0;
636 loc.line_start = i + 1;
637 } else {
638 loc.column += 1;
639 }
640 }
641 return loc;
642}
643
644pub fn tokenSlice(ast: Ast, token_index: TokenIndex) []const u8 {
645 return ast.tokens.get(token_index).slice(ast.utf8_source);
646}
647
648pub fn basicNodeSlice(ast: Ast, node: NodeIndex) []const u8 {
649 assert(ast.nodes.items(.tag)[node].isBasic());
650 const token = ast.nodes.items(.main_token)[node];
651 return ast.tokenSlice(token);
652}
653
654pub fn extraData(tree: Ast, index: usize, comptime T: type) T {
655 const fields = std.meta.fields(T);
656 var result: T = undefined;
657 var i: usize = 0;
658 inline for (fields) |field| {
659 @field(result, field.name) = switch (field.type) {
660 NodeIndex => tree.extra[index + i],
661 SubList => SubList{
662 .start = tree.extra[index + i],
663 .len = tree.extra[index + i + 1],
664 },
665 OptionalNodeIndex => @enumFromInt(tree.extra[index + i]),
666 OptionalTokenIndex => @enumFromInt(tree.extra[index + i]),
667 OptionalExtraIndex => @enumFromInt(tree.extra[index + i]),
668 else => @compileError("ExtraData type not supported: " ++ @typeName(field.type)),
669 };
670 i += switch (field.type) {
671 NodeIndex,
672 OptionalNodeIndex,
673 OptionalTokenIndex,
674 OptionalExtraIndex,
675 => 1,
676 SubList => 2,
677 else => comptime unreachable,
678 };
679 }
680 return result;
681}
682
683pub fn dumpToWriter(ast: Ast, writer: anytype) !void {
684 try writer.print("{}", .{Dump{ .ast = ast }});
685}
686
687pub const Dump = @import("Ast.Dump.zig");
688
689pub const Unused = enum(u32) { zero = 0 };
690pub const TokenIndex = u32;
691pub const ExtraIndex = u32;
692pub const NodeIndex = u32;
693
694const Ast = @This();
695const std = @import("std");
696const assert = std.debug.assert;
697
698const Tokenizer = @import("Tokenizer.zig");
699const Token = Tokenizer.Token;
700const TokenList = Tokenizer.Token.List;
701const ByteOffset = Tokenizer.ByteOffset;
src/compiler/Cache.zig created+9
......@@ -0,0 +1,9 @@
1const std = @import("std");
2
3/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
4pub const bin_digest_len = 16;
5pub const hex_digest_len = bin_digest_len * 2;
6pub const BinDigest = [bin_digest_len]u8;
7pub const HexDigest = [hex_digest_len]u8;
8
9pub const HashHelper = std.Build.Cache.HashHelper;
src/compiler/Comp.zig created+395
......@@ -0,0 +1,395 @@
1//! Compilation
2
3/// General purpose allocator
4gpa: Allocator,
5/// Stores all Type and Value objects.
6intern_pool: InternPool = .{},
7/// All files in the compilation
8files: std.StringArrayHashMapUnmanaged(*File) = .{},
9
10/// # of sema instructions in this compilation cycle
11analyzed_instructions: usize = 0,
12
13// TODO: consider computing this dynamicly per-link
14current_compile_decls: std.ArrayListUnmanaged(Decl.Index) = .{},
15
16pub fn init(comp: *Comp) !void {
17 try comp.intern_pool.init(comp.gpa);
18}
19
20pub fn deinit(comp: *Comp) void {
21 comp.intern_pool.deinit(comp.gpa);
22 for (comp.files.values()) |file| {
23 file.deinit(comp.gpa);
24 }
25 comp.files.deinit(comp.gpa);
26 comp.current_compile_decls.deinit(comp.gpa);
27}
28
29pub fn getFile(comp: *Comp, absolute_file_path: []const u8) !*File {
30 assert(std.fs.path.isAbsolute(absolute_file_path));
31 const gop = try comp.files.getOrPut(comp.gpa, absolute_file_path);
32 if (gop.found_existing) {
33 return gop.value_ptr.*;
34 }
35 const file = try comp.gpa.create(File);
36 file.* = .{
37 .file_path = absolute_file_path,
38 .path_digest = null,
39 .root_module = .none,
40 .status = .never_loader,
41 .source = null,
42 .ast = null,
43 .teg = null,
44 };
45 gop.value_ptr.* = file;
46 return file;
47}
48
49pub const Source = struct {
50 code: [:0]const u8,
51 stat: Stat,
52};
53
54pub const File = struct {
55 /// TODO: change this to a URL
56 file_path: []const u8,
57
58 root_module: Decl.Index.Optional,
59 status: enum {
60 never_loader,
61 /// A failure occurred during loading the file. Retrying might succeed.
62 retryable_failure,
63 /// Errors during parse
64 parse_failure,
65 /// Errors during transform
66 transform_failure,
67 /// Teg is valid
68 success,
69 },
70
71 source: ?Source,
72 ast: ?Ast,
73 teg: ?Teg,
74
75 path_digest: ?Cache.BinDigest,
76
77 pub fn getSource(file: *File, gpa: Allocator) !Source {
78 if (file.source) |source| {
79 return source;
80 }
81
82 // Keep track of inode, file size, mtime, hash so we can detect which files
83 // have been modified when an incremental update is requested.
84 var f = try std.fs.openFileAbsolute(file.file_path, .{});
85 defer f.close();
86
87 const stat = try f.stat();
88
89 if (stat.size > std.math.maxInt(u32))
90 return error.FileTooBig;
91
92 const source = try gpa.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0);
93 errdefer gpa.free(source);
94 const amt = try f.readAll(source);
95 if (amt != stat.size)
96 return error.UnexpectedEndOfFile;
97
98 // Here we do not modify stat fields because this function is the one
99 // used for error reporting. We need to keep the stat fields stale so that
100 // transformFile can know to regenerate Teg.
101
102 const loaded_source = Source{
103 .code = source,
104 .stat = .{
105 .size = stat.size,
106 .inode = stat.inode,
107 .mtime = stat.mtime,
108 },
109 };
110 file.source = loaded_source;
111 return loaded_source;
112 }
113
114 pub fn getAst(file: *File, gpa: Allocator) !Ast {
115 if (file.ast) |ast| {
116 return ast;
117 }
118
119 const ast = try Ast.parse(gpa, (try file.getSource(gpa)).code);
120 file.ast = ast;
121 return ast;
122 }
123
124 pub fn getTeg(file: *File, gpa: Allocator) !Teg {
125 if (file.teg) |teg| {
126 return teg;
127 }
128
129 const bin_digest = hash: {
130 var path_hash: Cache.HashHelper = .{};
131 path_hash.addBytes("0.1.0");
132 path_hash.addBytes(file.file_path);
133 var bin: Cache.BinDigest = undefined;
134 path_hash.hasher.final(&bin);
135 break :hash bin;
136 };
137 file.path_digest = bin_digest;
138
139 const teg = try Transform.transform(try file.getAst(gpa), gpa);
140 file.teg = teg;
141 return teg;
142 }
143
144 pub fn unloadSource(file: *File, gpa: Allocator) void {
145 if (file.source) |source| {
146 gpa.free(source.code);
147 file.source = null;
148 }
149 }
150
151 pub fn unloadAst(file: *File, gpa: Allocator) void {
152 if (file.ast) |*ast| {
153 ast.deinit(gpa);
154 file.ast = null;
155 }
156 }
157
158 pub fn unloadTeg(file: *File, gpa: Allocator) void {
159 if (file.teg) |*teg| {
160 teg.deinit(gpa);
161 file.teg = null;
162 }
163 }
164
165 pub fn deinit(file: *File, gpa: Allocator) void {
166 file.unloadTeg(gpa);
167 file.unloadAst(gpa);
168 file.unloadSource(gpa);
169 gpa.destroy(file);
170 }
171};
172
173pub const Decl = struct {
174 name: InternPool.NullTerminatedString,
175 val: Value,
176 /// The direct parent namespace of the Decl. In the case of the Decl
177 /// corresponding to a file, this is the namespace of the struct, since
178 /// there is no parent.
179 src_namespace: Namespace.Index,
180
181 /// The AST node index of this declaration.
182 /// Must be recomputed when the corresponding source file is modified.
183 // src_node: Ast.NodeIndex,
184 /// Line number corresponding to `src_node`. Stored separately so that source files
185 /// do not need to be loaded into memory in order to compute debug line numbers.
186 /// This value is absolute.
187 // src_line: u32,
188 /// Index of the ZIR `declaration` instruction from which this `Decl` was created.
189 /// For the root `Decl` of a `File` and legacy anonymous decls, this is `.none`.
190 teg_decl_index: InternPool.TrackedInst.Index.Optional,
191
192 /// Represents the "shallow" analysis status. For example, for decls that are functions,
193 /// the function type is analyzed with this set to `in_progress`, however, the semantic
194 /// analysis of the function body is performed with this value set to `success`. Functions
195 /// have their own analysis status field.
196 analysis: enum {
197 /// This Decl corresponds to an AST Node that has not been referenced yet, and therefore
198 /// because of Zig's lazy declaration analysis, it will remain unanalyzed until referenced.
199 unreferenced,
200 /// Semantic analysis for this Decl is running right now.
201 /// This state detects dependency loops.
202 in_progress,
203 /// The file corresponding to this Decl had a parse error or ZIR error.
204 /// There will be a corresponding ErrorMsg in Zcu.failed_files.
205 file_failure,
206 /// This Decl might be OK but it depends on another one which did not
207 /// successfully complete semantic analysis.
208 dependency_failure,
209 /// Semantic analysis failure.
210 /// There will be a corresponding ErrorMsg in Zcu.failed_decls.
211 sema_failure,
212 /// There will be a corresponding ErrorMsg in Zcu.failed_decls.
213 codegen_failure,
214 /// Sematic analysis and constant value codegen of this Decl has
215 /// succeeded. However, the Decl may be outdated due to an in-progress
216 /// update. Note that for a function, this does not mean codegen of the
217 /// function body succeded: that state is indicated by the function's
218 /// `analysis` field.
219 complete,
220 },
221
222 codegen: ?JsTree = null,
223
224 is_pub: bool,
225
226 /// What kind of a declaration is this.
227 kind: Kind,
228
229 pub fn deinit(decl: *Decl, gpa: Allocator) void {
230 defer decl.* = undefined;
231 if (decl.codegen) |*cg| cg.deinit(gpa);
232 }
233
234 pub const Kind = enum {
235 named,
236 };
237
238 pub const Index = InternPool.DeclIndex;
239};
240
241pub const Namespace = struct {
242 parent: Index.Optional,
243 file_scope: *File,
244 /// Direct children of the namespace.
245 /// Declaration order is preserved via entry order.
246 /// These are only declarations named directly by the AST; anonymous
247 /// declarations are not stored here.
248 decls: std.ArrayHashMapUnmanaged(Decl.Index, void, DeclContext, true) = .{},
249
250 pub const DeclContext = struct {
251 comp: *Comp,
252
253 pub fn hash(ctx: @This(), decl_index: Decl.Index) u32 {
254 const decl = ctx.comp.intern_pool.declPtr(decl_index);
255 return std.hash.uint32(@intFromEnum(decl.name));
256 }
257
258 pub fn eql(ctx: @This(), a_decl_index: Decl.Index, b_decl_index: Decl.Index, b_index: usize) bool {
259 _ = b_index;
260 const a_decl = ctx.comp.intern_pool.declPtr(a_decl_index);
261 const b_decl = ctx.comp.intern_pool.declPtr(b_decl_index);
262 return a_decl.name == b_decl.name;
263 }
264 };
265
266 pub const DeclNameAdapter = struct {
267 comp: *Comp,
268
269 pub fn hash(self: @This(), s: InternPool.NullTerminatedString) u32 {
270 _ = self;
271 return std.hash.uint32(@intFromEnum(s));
272 }
273
274 pub fn eql(self: @This(), a: InternPool.NullTerminatedString, b_decl_index: Decl.Index, b_index: usize) bool {
275 _ = b_index;
276 return a == self.comp.intern_pool.declPtr(b_decl_index).name;
277 }
278 };
279
280 pub fn lookupDirect(ns: *Namespace, comp: *Comp, name: InternPool.NullTerminatedString) ?Decl.Index {
281 return ns.decls.getKeyAdapted(name, Namespace.DeclNameAdapter{ .comp = comp });
282 }
283
284 pub const Index = InternPool.NamespaceIndex;
285};
286
287/// Index a module in a Teg.
288/// When a File's Teg is loaded, we call this on the root module.
289/// submodules are analyzed lazily.
290pub fn indexTegModule(comp: *Comp, file: *File, module_index: Teg.Inst.Index) !Namespace.Index {
291 const teg = file.teg.?;
292
293 log.info("Index {} @ {d}", .{ std.fmt.fmtSliceHexLower(file.path_digest.?[0..8]), @intFromEnum(module_index) });
294
295 const ns = try comp.intern_pool.createNamespace(comp.gpa, .{
296 .parent = .none,
297 .file_scope = file,
298 });
299 const ns_ptr = comp.intern_pool.namespacePtr(ns);
300 errdefer comp.intern_pool.destroyNamespace(comp.gpa, ns);
301
302 for (teg.moduleDeclarations(module_index)) |decl_index| {
303 const name = try comp.intern_pool.getOrPutString(
304 comp.gpa,
305 teg.declName(decl_index),
306 .no_embedded_nulls,
307 );
308 const tracked_inst = try comp.intern_pool.trackInst(comp.gpa, file, decl_index);
309 const decl = try comp.intern_pool.createDecl(comp.gpa, .{
310 .name = name,
311 .val = undefined,
312 .src_namespace = ns,
313 .teg_decl_index = tracked_inst.toOptional(),
314 .analysis = .unreferenced,
315 .is_pub = false, // todo:
316 .kind = .named,
317 });
318 try ns_ptr.decls.putNoClobberContext(comp.gpa, decl, void{}, .{ .comp = comp });
319 }
320
321 return ns;
322}
323
324pub fn semaEntryPoint(comp: *Comp, file: *File) !void {
325 const teg = try file.getTeg(comp.gpa);
326
327 const ns = try comp.indexTegModule(file, .main_module_inst);
328 const decl_index = comp.intern_pool.namespacePtr(ns).lookupDirect(comp, .empty) orelse
329 @panic("no top level code?");
330 const decl = comp.intern_pool.declPtr(decl_index);
331
332 assert(comp.current_compile_decls.items.len == 0);
333
334 var arena = std.heap.ArenaAllocator.init(comp.gpa);
335 defer arena.deinit();
336
337 var sema: Sema = .{
338 .comp = comp,
339 .gpa = comp.gpa,
340 .code = teg,
341 .arena = arena.allocator(),
342 .ip = &comp.intern_pool,
343 .decl = decl,
344 .decl_index = decl_index,
345 .namespace_index = ns,
346 };
347 try sema.init();
348 defer sema.deinit();
349 _ = try sema.analyzeDeclaration();
350 try sema.finalize();
351}
352
353pub fn codeGenDecl(c: *Comp, decl: *Decl, air: *const Air) !void {
354 const t = tracer.trace(@src());
355 defer t.end();
356
357 std.debug.print("Start Codegen\n", .{});
358 defer std.debug.print("End Codegen\n", .{});
359
360 var arena_state = std.heap.ArenaAllocator.init(c.gpa);
361 defer arena_state.deinit();
362 const arena = arena_state.allocator();
363
364 var cg = try JsNodeGen.init(c.gpa, arena, &c.intern_pool, air);
365 errdefer cg.deinit();
366 try cg.build();
367 decl.codegen = try cg.finalize();
368}
369
370pub fn link(c: *Comp, writer: anytype) !void {
371 try JsLink.linkJsCode(c, c.current_compile_decls.items, writer);
372}
373
374const Comp = @This();
375
376const std = @import("std");
377const Allocator = std.mem.Allocator;
378const assert = std.debug.assert;
379const Stat = std.Build.Cache.File.Stat;
380const log = std.log.scoped(.comp);
381
382const InternPool = @import("InternPool.zig");
383
384const Ast = @import("Ast.zig");
385const Air = @import("Air.zig");
386const Teg = @import("Teg.zig");
387const Cache = @import("Cache.zig");
388const Value = @import("Value.zig");
389const Sema = @import("Sema.zig");
390const Transform = @import("Transform.zig");
391const JsNodeGen = @import("JsNodeGen.zig");
392const JsTree = @import("JsTree.zig");
393const JsLink = @import("JsLink.zig");
394
395const tracer = @import("tracer");
src/compiler/InternPool.zig created+1018
......@@ -0,0 +1,1018 @@
1//! Large parts of this file are based on the Zig compiler's InternPool.
2//! https://github.com/ziglang/zig/blob/0.12.0/src/InternPool.zig
3
4/// Maps `Key` to `Index`. `Key` objects are not stored anywhere; they are
5/// constructed lazily.
6map: std.AutoArrayHashMapUnmanaged(void, void) = .{},
7items: std.MultiArrayList(Item) = .{},
8extra: ExtraStore = .{},
9
10/// In order to store references to strings in fewer bytes, we copy all
11/// string bytes into here. String bytes can be null. It is up to whomever
12/// is referencing the data here whether they want to store both index and length,
13/// thus allowing null bytes, or store only index, and use null-termination. The
14/// `string_bytes` array is agnostic to either usage.
15string_bytes: std.ArrayListUnmanaged(u8) = .{},
16
17/// Used for finding the index inside `string_bytes`.
18string_table: std.HashMapUnmanaged(
19 u32,
20 void,
21 std.hash_map.StringIndexContext,
22 std.hash_map.default_max_load_percentage,
23) = .{},
24
25/// An index into `tracked_insts` gives a reference to a single ZIR instruction which
26/// persists across incremental updates.
27tracked_insts: std.AutoArrayHashMapUnmanaged(TrackedInst, void) = .{},
28
29/// Rather than allocating Decl objects with an Allocator, we instead allocate
30/// them with this SegmentedList. This provides four advantages:
31/// * Stable memory so that one thread can access a Decl object while another
32/// thread allocates additional Decl objects from this list.
33/// * It allows us to use u32 indexes to reference Decl objects rather than
34/// pointers, saving memory in Type, Value, and dependency sets.
35/// * Using integers to reference Decl objects rather than pointers makes
36/// serialization trivial.
37/// * It provides a unique integer to be used for anonymous symbol names, avoiding
38/// multi-threaded contention on an atomic counter.
39allocated_decls: std.SegmentedList(Comp.Decl, 0) = .{},
40/// When a Decl object is freed from `allocated_decls`, it is pushed into this stack.
41decls_free_list: std.ArrayListUnmanaged(DeclIndex) = .{},
42
43/// Same pattern as with `allocated_decls`.
44allocated_namespaces: std.SegmentedList(Comp.Namespace, 0) = .{},
45/// Same pattern as with `decls_free_list`.
46namespaces_free_list: std.ArrayListUnmanaged(NamespaceIndex) = .{},
47
48/// How many items in the InternPool are statically known.
49/// This is specified with an integer literal and a corresponding comptime
50/// assert below to break an unfortunate and arguably incorrect dependency loop
51/// when compiling.
52pub const static_len = Teg.Inst.Index.static_len;
53comptime {
54 // @compileLog(static_len, static_keys.len);
55 assert(static_len == static_keys.len);
56}
57
58pub const Item = struct {
59 tag: Tag,
60 data: EncodedData,
61
62 pub fn encode(comptime tag: Tag, data: tag.Data()) Item {
63 return .{ .tag = tag, .data = EncodedData.encode(tag, data) };
64 }
65};
66
67pub const ExtraStore = dod.ExtraStore(.intern_pool, .@"var");
68pub const EncodedData = dod.EncodedData(Tag, u32);
69pub const ExtraIndex = ExtraStore.Index;
70
71pub const Tag = enum(u8) {
72 /// This special tag represents a value which was removed from this pool via
73 /// `InternPool.remove`. The item remains allocated to preserve indices, but
74 /// lookups will consider it not equal to any other item, and all queries
75 /// assert not this tag. `data` is unused.
76 removed,
77
78 /// A function type
79 /// data is an extra index into a `FuncType`
80 type_function,
81
82 // --- before this are types ---
83 // --- past this are values ---
84
85 /// A type that can be represented with only an enum tag.
86 /// data is SimpleType enum value.
87 simple_type,
88 /// A value that can be represented with only an enum tag.
89 /// data is SimpleValue enum value.
90 simple_value,
91 /// An int32 value
92 /// data is i32
93 int32,
94 /// A float value
95 /// data is an extra index into an `Float`
96 float,
97 /// A function
98 /// data is an extra index into a `FuncDecl`
99 func,
100 /// A string
101 /// data is an extra index into a `String`
102 string,
103
104 pub fn Data(comptime tag: Tag) type {
105 return switch (tag) {
106 .removed,
107 => unreachable, // does not have a data
108
109 .simple_type => SimpleType,
110 .simple_value => SimpleValue,
111 .int32 => i32,
112
113 else => ExtraIndex(tag.Payload()),
114 };
115 }
116
117 /// When `data` is an extra index, this is the type of the payload.
118 pub fn Payload(comptime tag: Tag) type {
119 return switch (tag) {
120 .removed,
121 .simple_type,
122 .simple_value,
123 .int32,
124 => unreachable, // does not have a payload
125 .float => Float,
126 .func => FuncDecl,
127 .type_function => TypeFunction,
128 .string => String.WithLen,
129 };
130 }
131
132 /// Trailing: N/A
133 pub const FuncDecl = struct {
134 analysis: FuncAnalysis,
135 owner_decl: DeclIndex,
136 ty: Index,
137 teg_body_inst: TrackedInst.Index,
138 };
139
140 /// Trailing:
141 /// 0. param_type: Index for each params_len
142 pub const TypeFunction = struct {
143 params_len: u32,
144 return_type: Index,
145 };
146};
147
148pub const Index = enum(u32) {
149 pub const first_type: Index = .opaque_type;
150 pub const last_type: Index = .noreturn_type;
151 pub const first_value: Index = .void_value;
152 pub const last_value: Index = .empty_string;
153
154 opaque_type,
155 bool_type,
156 int32_type,
157 float_type,
158 string_type,
159 void_type,
160 noreturn_type,
161 type_type,
162
163 void_value,
164 bool_true,
165 bool_false,
166 int32_zero,
167 float_zero,
168 nan,
169 empty_string,
170
171 none = std.math.maxInt(u32),
172 _,
173
174 /// An array of `Index` existing within the `extra` array.
175 /// This type exists to provide a struct with lifetime that is
176 /// not invalidated when items are added to the `InternPool`.
177 pub const Slice = struct {
178 start: u32,
179 len: u32,
180
181 pub fn get(slice: Slice, ip: *const InternPool) []Index {
182 return @ptrCast(ip.extra.list.items[slice.start..][0..slice.len]);
183 }
184 };
185
186 /// Used for a map of `Index` values to the index within a list of `Index` values.
187 const Adapter = struct {
188 indexes: []const Index,
189
190 pub fn eql(ctx: @This(), a: Index, b_void: void, b_map_index: usize) bool {
191 _ = b_void;
192 return a == ctx.indexes[b_map_index];
193 }
194
195 pub fn hash(ctx: @This(), a: Index) u32 {
196 _ = ctx;
197 return std.hash.uint32(@intFromEnum(a));
198 }
199 };
200
201 pub fn toAir(index: Index) Air.Inst.Ref {
202 return .{ .kind = .value, .data = @intCast(@intFromEnum(index)) };
203 }
204
205 pub fn toKey(index: Index, ip: *const InternPool) Key {
206 return ip.indexToKey(index);
207 }
208};
209
210pub const static_keys = [_]Key{
211 SimpleType.@"opaque".key(),
212 SimpleType.bool.key(),
213 SimpleType.int32.key(),
214 SimpleType.float.key(),
215 SimpleType.string.key(),
216 SimpleType.void.key(),
217 SimpleType.noreturn.key(),
218 SimpleType.type.key(),
219
220 SimpleValue.void.key(),
221 SimpleValue.true.key(),
222 SimpleValue.false.key(),
223 .{ .int32 = 0 },
224 .{ .float = 0.0 },
225 .{ .float = Float.pure_nan.decode() },
226 .{ .string = "" },
227};
228
229/// State that is mutable during semantic analysis. This data is not used for
230/// equality or hashing.
231pub const FuncAnalysis = packed struct(u32) {
232 state: State,
233
234 _: u24 = 0,
235
236 pub const State = enum(u8) {
237 /// This function has not yet undergone analysis, because we have not
238 /// seen a potential runtime call. It may be analyzed in future.
239 none,
240 /// Analysis for this function has been queued, but not yet completed.
241 queued,
242 /// This function intentionally only has ZIR generated because it is marked
243 /// inline, which means no runtime version of the function will be generated.
244 inline_only,
245 in_progress,
246 /// There will be a corresponding ErrorMsg in Module.failed_decls
247 sema_failure,
248 /// This function might be OK but it depends on another Decl which did not
249 /// successfully complete semantic analysis.
250 dependency_failure,
251 /// There will be a corresponding ErrorMsg in Module.failed_decls.
252 /// Indicates that semantic analysis succeeded, but code generation for
253 /// this function failed.
254 codegen_failure,
255 /// Semantic analysis and code generation of this function succeeded.
256 success,
257 };
258};
259
260/// An index into `string_bytes`.
261pub const String = enum(u32) {
262 /// An empty string.
263 empty = 0,
264 _,
265
266 pub const WithLen = struct {
267 string: String,
268 len: u32,
269
270 pub fn toSlice(self: WithLen, ip: *const InternPool) []const u8 {
271 return ip.string_bytes.items[@intFromEnum(self.string)..][0..@intCast(self.len)];
272 }
273
274 pub fn tryToNullTerminatedString(string: WithLen, ip: *InternPool, gpa: Allocator) !NullTerminatedString {
275 return string.string.tryToNullTerminatedString(string.len, ip, gpa);
276 }
277
278 pub fn toNullTerminatedString(string: WithLen, ip: *const InternPool) NullTerminatedString {
279 return string.string.tryToNullTerminatedString(string.len, ip);
280 }
281 };
282
283 pub fn toSlice(string: String, len: u64, ip: *const InternPool) []const u8 {
284 return ip.string_bytes.items[@intFromEnum(string)..][0..@intCast(len)];
285 }
286
287 pub fn withLen(string: String, len: u64) WithLen {
288 return .{ .string = string, .len = @intCast(len) };
289 }
290
291 pub fn at(string: String, index: u64, ip: *const InternPool) u8 {
292 return ip.string_bytes.items[@intCast(@intFromEnum(string) + index)];
293 }
294
295 pub fn toNullTerminatedString(string: String, len: u64, ip: *const InternPool) NullTerminatedString {
296 assert(std.mem.indexOfScalar(u8, string.toSlice(len, ip), 0) == null);
297 assert(string.at(len, ip) == 0);
298 return @enumFromInt(@intFromEnum(string));
299 }
300
301 pub fn tryToNullTerminatedString(
302 string: String,
303 len: u64,
304 ip: *InternPool,
305 gpa: Allocator,
306 ) !NullTerminatedString {
307 const slice = string.toSlice(len, ip);
308 if (std.mem.indexOfScalar(u8, slice, 0)) |_| {
309 return error.HasNulls;
310 }
311 if (string.at(len, ip) == 0) {
312 return @enumFromInt(@intFromEnum(string));
313 }
314 return ip.getOrPutString(gpa, slice, .no_embedded_nulls);
315 }
316};
317
318pub const NullTerminatedString = enum(u32) {
319 empty = 0,
320 _,
321
322 pub fn toString(self: NullTerminatedString) String {
323 return @enumFromInt(@intFromEnum(self));
324 }
325
326 // pub fn toOptional(self: NullTerminatedString) Optional{
327 // return @enumFromInt(@intFromEnum(self));
328 // }
329
330 pub fn toSlice(string: NullTerminatedString, ip: *const InternPool) [:0]const u8 {
331 const slice = ip.string_bytes.items[@intFromEnum(string)..];
332 return slice[0..std.mem.indexOfScalar(u8, slice, 0).? :0];
333 }
334
335 pub fn length(string: NullTerminatedString, ip: *const InternPool) u32 {
336 return @intCast(string.toSlice(ip).len);
337 }
338
339 pub fn eqlSlice(string: NullTerminatedString, slice: []const u8, ip: *const InternPool) bool {
340 return std.mem.eql(u8, string.toSlice(ip), slice);
341 }
342
343 const Adapter = struct {
344 strings: []const NullTerminatedString,
345
346 pub fn eql(ctx: @This(), a: NullTerminatedString, b_void: void, b_map_index: usize) bool {
347 _ = b_void;
348 return a == ctx.strings[b_map_index];
349 }
350
351 pub fn hash(ctx: @This(), a: NullTerminatedString) u32 {
352 _ = ctx;
353 return std.hash.uint32(@intFromEnum(a));
354 }
355 };
356
357 const FormatData = struct {
358 string: NullTerminatedString,
359 ip: *const InternPool,
360 };
361 fn format(
362 data: FormatData,
363 comptime specifier: []const u8,
364 _: std.fmt.FormatOptions,
365 writer: anytype,
366 ) @TypeOf(writer).Error!void {
367 const slice = data.string.toSlice(data.ip);
368 if (comptime std.mem.eql(u8, specifier, "")) {
369 try writer.writeAll(slice);
370 } else if (comptime std.mem.eql(u8, specifier, "i")) {
371 try writer.print("{p}", .{std.zig.fmtId(slice)});
372 } else @compileError("invalid format string '" ++ specifier ++ "' for '" ++ @typeName(NullTerminatedString) ++ "'");
373 }
374
375 pub fn fmt(self: NullTerminatedString, ip: *const InternPool) std.fmt.Formatter(format) {
376 return .{ .data = .{ .string = self, .ip = ip } };
377 }
378};
379
380pub const TrackedInst = extern struct {
381 path_digest: Cache.BinDigest,
382 inst: Teg.Inst.Index,
383
384 comptime {
385 // The fields should be tightly packed. See also serialiation logic in `Compilation.saveState`.
386 assert(@sizeOf(@This()) == Cache.bin_digest_len + @sizeOf(Teg.Inst.Index));
387 }
388
389 pub const Index = enum(u32) {
390 _,
391 pub fn resolve(i: TrackedInst.Index, ip: *const InternPool) Teg.Inst.Index {
392 return ip.tracked_insts.keys()[@intFromEnum(i)].inst;
393 }
394 pub fn toOptional(i: TrackedInst.Index) Optional {
395 return @enumFromInt(@intFromEnum(i));
396 }
397 pub const Optional = enum(u32) {
398 none = std.math.maxInt(u32),
399 _,
400 pub fn unwrap(opt: Optional) ?TrackedInst.Index {
401 return switch (opt) {
402 .none => null,
403 _ => @enumFromInt(@intFromEnum(opt)),
404 };
405 }
406 };
407 };
408};
409
410pub const DeclIndex = enum(u32) {
411 _,
412
413 pub fn toOptional(i: DeclIndex) Optional {
414 return @enumFromInt(@intFromEnum(i));
415 }
416
417 pub const Optional = enum(u32) {
418 none = std.math.maxInt(u32),
419 _,
420
421 pub fn init(oi: ?DeclIndex) Optional {
422 return @enumFromInt(@intFromEnum(oi orelse return .none));
423 }
424
425 pub fn unwrap(oi: Optional) ?DeclIndex {
426 if (oi == .none) return null;
427 return @enumFromInt(@intFromEnum(oi));
428 }
429 };
430};
431
432pub const NamespaceIndex = enum(u32) {
433 _,
434
435 pub fn toOptional(i: DeclIndex) Optional {
436 return @enumFromInt(@intFromEnum(i));
437 }
438
439 pub const Optional = enum(u32) {
440 none = std.math.maxInt(u32),
441 _,
442
443 pub fn init(oi: ?NamespaceIndex) Optional {
444 return @enumFromInt(@intFromEnum(oi orelse return .none));
445 }
446
447 pub fn unwrap(oi: Optional) ?NamespaceIndex {
448 if (oi == .none) return null;
449 return @enumFromInt(@intFromEnum(oi));
450 }
451 };
452};
453
454/// A single value captured in the closure of a namespace type. This is not a plain
455/// `Index` because we must differentiate between the following cases:
456/// * runtime-known value (where we store the type)
457/// * comptime-known value (where we store the value)
458/// * decl val (so that we can analyze the value lazily)
459/// * decl ref (so that we can analyze the reference lazily)
460pub const CaptureValue = packed struct(u32) {
461 tag: enum(u2) { @"comptime", runtime, decl_val, decl_ref },
462 idx: u30,
463
464 pub fn wrap(val: Unwrapped) CaptureValue {
465 return switch (val) {
466 .@"comptime" => |i| .{ .tag = .@"comptime", .idx = @intCast(@intFromEnum(i)) },
467 .runtime => |i| .{ .tag = .runtime, .idx = @intCast(@intFromEnum(i)) },
468 .decl_val => |i| .{ .tag = .decl_val, .idx = @intCast(@intFromEnum(i)) },
469 .decl_ref => |i| .{ .tag = .decl_ref, .idx = @intCast(@intFromEnum(i)) },
470 };
471 }
472 pub fn unwrap(val: CaptureValue) Unwrapped {
473 return switch (val.tag) {
474 .@"comptime" => .{ .@"comptime" = @enumFromInt(val.idx) },
475 .runtime => .{ .runtime = @enumFromInt(val.idx) },
476 .decl_val => .{ .decl_val = @enumFromInt(val.idx) },
477 .decl_ref => .{ .decl_ref = @enumFromInt(val.idx) },
478 };
479 }
480
481 pub const Unwrapped = union(enum) {
482 /// Index refers to the value.
483 @"comptime": Index,
484 /// Index refers to the type.
485 runtime: Index,
486 decl_val: DeclIndex,
487 decl_ref: DeclIndex,
488 };
489
490 pub const Slice = struct {
491 start: u32,
492 len: u32,
493 pub fn get(slice: Slice, ip: *const InternPool) []CaptureValue {
494 return @ptrCast(ip.extra.items[slice.start..][0..slice.len]);
495 }
496 };
497};
498
499const SimpleValue = enum {
500 true,
501 false,
502 void,
503
504 pub fn key(self: SimpleValue) Key {
505 return .{ .simple_value = self };
506 }
507};
508
509const SimpleType = enum {
510 @"opaque",
511 int32,
512 float,
513 string,
514 bool,
515 void,
516 noreturn,
517 type,
518
519 pub fn key(self: SimpleType) Key {
520 return .{ .simple_type = self };
521 }
522};
523
524pub const Key = union(enum) {
525 int32: i32,
526 float: f64,
527 string: []const u8,
528 simple_value: SimpleValue,
529 simple_type: SimpleType,
530 type_function: TypeFunction,
531 func: Func,
532
533 const Adapter = struct {
534 intern_pool: *const InternPool,
535
536 pub fn eql(ctx: @This(), a: Key, b_void: void, b_map_index: usize) bool {
537 _ = b_void;
538 if (ctx.intern_pool.items.items(.tag)[b_map_index] == .removed) return false;
539 return ctx.intern_pool.indexToKey(@enumFromInt(b_map_index)).eql(a, ctx.intern_pool);
540 }
541
542 pub fn hash(ctx: @This(), a: Key) u32 {
543 return a.hash32(ctx.intern_pool);
544 }
545 };
546
547 pub const TypeFunction = struct {
548 param_types: []Index,
549 return_type: Index,
550
551 pub fn hash(self: TypeFunction, hasher: *Hash) void {
552 const t = tracer.trace(@src());
553 defer t.end();
554
555 for (self.param_types) |param_type| {
556 std.hash.autoHash(hasher, param_type);
557 }
558 std.hash.autoHash(hasher, self.return_type);
559 }
560
561 pub fn eql(a: TypeFunction, b: TypeFunction) bool {
562 return std.mem.eql(Index, a.param_types, b.param_types) and
563 a.return_type == b.return_type;
564 }
565 };
566
567 pub const Func = struct {
568 ty: Index,
569 owner_decl: DeclIndex,
570 teg_body_inst: TrackedInst.Index,
571
572 pub fn hash(self: Func, hasher: *Hash) void {
573 std.hash.autoHash(hasher, self.ty);
574 std.hash.autoHash(hasher, self.owner_decl);
575 std.hash.autoHash(hasher, self.teg_body_inst);
576 }
577 };
578
579 pub fn hash32(key: Key, ip: *const InternPool) u32 {
580 return @truncate(key.hash64(ip));
581 }
582
583 pub fn hash64(key: Key, ip: *const InternPool) u64 {
584 const t = tracer.trace(@src());
585 defer t.end();
586
587 _ = ip; // autofix
588 const asBytes = std.mem.asBytes;
589 const KeyTag = @typeInfo(Key).Union.tag_type.?;
590 const seed = @intFromEnum(@as(KeyTag, key));
591
592 return switch (key) {
593 inline .int32,
594 .simple_value,
595 .simple_type,
596 => |x| Hash.hash(seed, asBytes(&x)),
597 .string,
598 => |x| Hash.hash(seed, x),
599 .float => |x| Hash.hash(seed, asBytes(&Float.assertPure(x))),
600 .type_function => |x| {
601 var hasher = Hash.init(seed);
602 x.hash(&hasher);
603 return hasher.final();
604 },
605 .func => |x| {
606 var hasher = Hash.init(seed);
607 x.hash(&hasher);
608 return hasher.final();
609 },
610 };
611 }
612
613 pub fn eql(a: Key, b: Key, ip: *const InternPool) bool {
614 const t = tracer.trace(@src());
615 defer t.end();
616
617 _ = ip; // autofix
618 const KeyTag = @typeInfo(Key).Union.tag_type.?;
619 const a_tag: KeyTag = a;
620 const b_tag: KeyTag = b;
621 if (a_tag != b_tag) return false;
622 return switch (a) {
623 inline .int32,
624 .simple_value,
625 .simple_type,
626 => |x, tag| x == @field(b, @tagName(tag)),
627 .string,
628 => |x| std.mem.eql(u8, x, b.string),
629 .float => |x| Float.assertPure(x) == Float.assertPure(b.float),
630 .type_function => |x| x.eql(b.type_function),
631 .func => |a_info| {
632 const b_info = b.func;
633 if (a_info.owner_decl != b_info.owner_decl)
634 return false;
635 if (a_info.ty == b_info.ty)
636 return true;
637
638 // TODO: zig has something extra here for inferred error sets
639 return false;
640 },
641 };
642 }
643};
644
645const EmbeddedNulls = enum {
646 no_embedded_nulls,
647 maybe_embedded_nulls,
648
649 const OptionalNullTerminatedString = @compileError("TODO");
650 const OptionalString = @compileError("TODO");
651
652 fn StringType(comptime embedded_nulls: EmbeddedNulls) type {
653 return switch (embedded_nulls) {
654 .no_embedded_nulls => NullTerminatedString,
655 .maybe_embedded_nulls => String,
656 };
657 }
658
659 fn OptionalStringType(comptime embedded_nulls: EmbeddedNulls) type {
660 return switch (embedded_nulls) {
661 .no_embedded_nulls => OptionalNullTerminatedString,
662 .maybe_embedded_nulls => OptionalString,
663 };
664 }
665};
666
667// implementations:
668
669/// To construct, instantiate `.{ .gpa = gpa }` then call this.
670pub fn init(ip: *InternPool, gpa: Allocator) !void {
671 const t = tracer.trace(@src());
672 defer t.end();
673
674 assert(ip.items.len == 0);
675
676 // Reserve string index 0 for an empty string.
677 assert((try ip.getOrPutString(gpa, "", .no_embedded_nulls)) == .empty);
678
679 // So that we can use `catch unreachable` below.
680 try ip.items.ensureUnusedCapacity(gpa, static_keys.len);
681 try ip.map.ensureUnusedCapacity(gpa, static_keys.len);
682 try ip.extra.list.ensureUnusedCapacity(gpa, static_keys.len);
683
684 // This inserts all the statically-known values into the intern pool in the
685 // order expected.
686 for (static_keys) |key| {
687 _ = ip.get(gpa, key) catch unreachable;
688 }
689
690 if (std.debug.runtime_safety) {
691 // Sanity check.
692 assert(ip.indexToKey(.bool_true).simple_value == .true);
693 assert(ip.indexToKey(.bool_false).simple_value == .false);
694 }
695
696 assert(ip.items.len == static_keys.len);
697}
698
699pub fn deinit(ip: *InternPool, gpa: Allocator) void {
700 const t = tracer.trace(@src());
701 defer t.end();
702
703 ip.map.deinit(gpa);
704 ip.items.deinit(gpa);
705 ip.extra.deinit(gpa);
706 // ip.limbs.deinit(gpa);
707 ip.string_bytes.deinit(gpa);
708
709 var decls = ip.allocated_decls.iterator(0);
710 while (decls.next()) |decl| {
711 decl.deinit(gpa);
712 }
713
714 ip.allocated_decls.deinit(gpa);
715 ip.decls_free_list.deinit(gpa);
716
717 var namespaces = ip.allocated_namespaces.iterator(0);
718 while (namespaces.next()) |ns| {
719 ns.decls.deinit(gpa);
720 }
721
722 ip.namespaces_free_list.deinit(gpa);
723 ip.allocated_namespaces.deinit(gpa);
724
725 // for (ip.maps.items) |*map| map.deinit(gpa);
726 // ip.maps.deinit(gpa);
727
728 ip.string_table.deinit(gpa);
729
730 ip.tracked_insts.deinit(gpa);
731
732 // ip.src_hash_deps.deinit(gpa);
733 // ip.decl_val_deps.deinit(gpa);
734 // ip.func_ies_deps.deinit(gpa);
735 // ip.namespace_deps.deinit(gpa);
736 // ip.namespace_name_deps.deinit(gpa);
737
738 // ip.first_dependency.deinit(gpa);
739
740 // ip.dep_entries.deinit(gpa);
741 // ip.free_dep_entries.deinit(gpa);
742
743 ip.* = undefined;
744}
745
746pub fn indexToKey(ip: *const InternPool, index: Index) Key {
747 assert(index != .none);
748 const item: Item = ip.items.get(@intFromEnum(index));
749 const data = item.data;
750 return switch (item.tag) {
751 .removed => unreachable,
752
753 .type_function => .{
754 .type_function = ip.extraTypeFunction(data.decode(.type_function, item.tag)),
755 },
756
757 .simple_type => .{ .simple_type = data.decode(.simple_type, item.tag) },
758 .simple_value => .{ .simple_value = data.decode(.simple_value, item.tag) },
759 .int32 => .{ .int32 = data.decode(.int32, item.tag) },
760 .float => .{ .float = data.decode(.float, item.tag).get(ip).decode() },
761 .string => .{ .string = data.decode(.string, item.tag).get(ip).toSlice(ip) },
762 .func => {
763 const func = data.decode(.func, item.tag).get(ip);
764 return .{ .func = .{
765 .ty = func.ty,
766 .owner_decl = func.owner_decl,
767 .teg_body_inst = func.teg_body_inst,
768 } };
769 },
770 };
771}
772
773pub fn get(ip: *InternPool, gpa: Allocator, key: Key) !Index {
774 const t = tracer.traceFmt(@src(), "{s}", .{@tagName(key)});
775 defer t.end();
776
777 const adapter: Key.Adapter = .{ .intern_pool = ip };
778 const gop = try ip.map.getOrPutAdapted(gpa, key, adapter);
779 if (gop.found_existing) return @enumFromInt(gop.index);
780 if (ip.items.len >= std.math.maxInt(u31) - 1) {
781 return error.TooManyValues;
782 }
783
784 const t2 = tracer.traceFmt(@src(), "(insert)", .{});
785 defer t2.end();
786
787 try ip.items.ensureUnusedCapacity(gpa, 1);
788 const item: Item = switch (key) {
789 inline .int32,
790 .simple_type,
791 .simple_value,
792 => |s, tag| Item.encode(@field(Tag, @tagName(tag)), s),
793 .string => |s| Item.encode(
794 .string,
795 try ip.extra.append(
796 gpa,
797 String.WithLen,
798 (try ip.getOrPutString(gpa, s, .maybe_embedded_nulls)).withLen(s.len),
799 ),
800 ),
801 .float => |f| Item.encode(
802 .float,
803 try ip.extra.append(gpa, Float, Float.encode(f)),
804 ),
805 .func => |f| brk: {
806 const extra = try ip.extra.append(gpa, Tag.FuncDecl, .{
807 .analysis = .{
808 .state = .none,
809 },
810 .owner_decl = f.owner_decl,
811 .ty = f.ty,
812 .teg_body_inst = f.teg_body_inst,
813 });
814
815 break :brk Item.encode(.func, extra);
816 },
817 .type_function => |f| brk: {
818 const extra, var trail = try ip.extra.appendTrailing(gpa, Tag.TypeFunction, .{
819 .params_len = @intCast(f.param_types.len),
820 .return_type = f.return_type,
821 }, f.param_types.len);
822 defer trail.end();
823 trail.writeSlice(f.param_types);
824 break :brk Item.encode(.type_function, extra);
825 },
826 };
827 ip.items.appendAssumeCapacity(item);
828 const index: Index = @enumFromInt(ip.items.len - 1);
829 if (std.debug.runtime_safety) {
830 assert(ip.indexToKey(index).eql(key, ip));
831 }
832 return index;
833}
834
835fn extraTypeFunction(
836 ip: *const InternPool,
837 extra_index: ExtraIndex(Tag.TypeFunction),
838) Key.TypeFunction {
839 const type_function, var trail = extra_index.getTrail(ip);
840 const param_types = trail.readInternPoolIndexSlice(type_function.params_len);
841 const result: Key.TypeFunction = .{
842 .return_type = type_function.return_type,
843 .param_types = param_types.get(ip),
844 };
845 return result;
846}
847
848pub fn getOrPutString(
849 ip: *InternPool,
850 gpa: Allocator,
851 slice: []const u8,
852 comptime embedded_nulls: EmbeddedNulls,
853) Allocator.Error!embedded_nulls.StringType() {
854 const t = tracer.trace(@src());
855 defer t.end();
856
857 try ip.string_bytes.ensureUnusedCapacity(gpa, slice.len + 1);
858 ip.string_bytes.appendSliceAssumeCapacity(slice);
859 ip.string_bytes.appendAssumeCapacity(0);
860 return ip.getOrPutTrailingString(gpa, slice.len + 1, embedded_nulls);
861}
862
863fn getOrPutTrailingString(
864 ip: *InternPool,
865 gpa: Allocator,
866 len: usize,
867 comptime embedded_nulls: EmbeddedNulls,
868) Allocator.Error!embedded_nulls.StringType() {
869 const t = tracer.trace(@src());
870 defer t.end();
871
872 const string_bytes = &ip.string_bytes;
873 const str_index: u32 = @intCast(string_bytes.items.len - len);
874 if (len > 0 and string_bytes.getLast() == 0) {
875 _ = string_bytes.pop();
876 } else {
877 try string_bytes.ensureUnusedCapacity(gpa, 1);
878 }
879 const key: []const u8 = string_bytes.items[str_index..];
880 const has_embedded_null = std.mem.indexOfScalar(u8, key, 0) != null;
881 switch (embedded_nulls) {
882 .no_embedded_nulls => assert(!has_embedded_null),
883 .maybe_embedded_nulls => if (has_embedded_null) {
884 string_bytes.appendAssumeCapacity(0);
885 return @enumFromInt(str_index);
886 },
887 }
888 const gop = try ip.string_table.getOrPutContextAdapted(gpa, key, std.hash_map.StringIndexAdapter{
889 .bytes = string_bytes,
890 }, std.hash_map.StringIndexContext{
891 .bytes = string_bytes,
892 });
893 if (gop.found_existing) {
894 string_bytes.shrinkRetainingCapacity(str_index);
895 return @enumFromInt(gop.key_ptr.*);
896 } else {
897 gop.key_ptr.* = str_index;
898 string_bytes.appendAssumeCapacity(0);
899 return @enumFromInt(str_index);
900 }
901}
902
903pub fn trackInst(ip: *InternPool, gpa: Allocator, file: *Comp.File, inst: Teg.Inst.Index) Allocator.Error!TrackedInst.Index {
904 const t = tracer.trace(@src());
905 defer t.end();
906
907 const key: TrackedInst = .{
908 .path_digest = file.path_digest.?,
909 .inst = inst,
910 };
911 const gop = try ip.tracked_insts.getOrPut(gpa, key);
912 return @enumFromInt(gop.index);
913}
914
915pub fn declPtr(ip: *InternPool, index: DeclIndex) *Comp.Decl {
916 return ip.allocated_decls.at(@intFromEnum(index));
917}
918
919pub fn declPtrConst(ip: *const InternPool, index: DeclIndex) *const Comp.Decl {
920 return ip.allocated_decls.at(@intFromEnum(index));
921}
922
923pub fn namespacePtr(ip: *InternPool, index: NamespaceIndex) *Comp.Namespace {
924 return ip.allocated_namespaces.at(@intFromEnum(index));
925}
926
927pub fn createDecl(
928 ip: *InternPool,
929 gpa: Allocator,
930 initialization: Comp.Decl,
931) Allocator.Error!DeclIndex {
932 const t = tracer.trace(@src());
933 defer t.end();
934
935 if (ip.decls_free_list.popOrNull()) |index| {
936 ip.allocated_decls.at(@intFromEnum(index)).* = initialization;
937 return index;
938 }
939 const ptr = try ip.allocated_decls.addOne(gpa);
940 ptr.* = initialization;
941 return @enumFromInt(ip.allocated_decls.len - 1);
942}
943
944pub fn destroyDecl(ip: *InternPool, gpa: Allocator, index: DeclIndex) void {
945 const t = tracer.trace(@src());
946 defer t.end();
947
948 ip.declPtr(index).* = undefined;
949 ip.decls_free_list.append(gpa, index) catch {
950 // In order to keep `destroyDecl` a non-fallible function, we ignore memory
951 // allocation failures here, instead leaking the Decl until garbage collection.
952 };
953}
954
955pub fn createNamespace(
956 ip: *InternPool,
957 gpa: Allocator,
958 initialization: Comp.Namespace,
959) Allocator.Error!NamespaceIndex {
960 const t = tracer.trace(@src());
961 defer t.end();
962
963 if (ip.namespaces_free_list.popOrNull()) |index| {
964 ip.allocated_namespaces.at(@intFromEnum(index)).* = initialization;
965 return index;
966 }
967 const ptr = try ip.allocated_namespaces.addOne(gpa);
968 ptr.* = initialization;
969 return @enumFromInt(ip.allocated_namespaces.len - 1);
970}
971
972pub fn destroyNamespace(ip: *InternPool, gpa: Allocator, index: NamespaceIndex) void {
973 const t = tracer.trace(@src());
974 defer t.end();
975
976 ip.namespacePtr(index).* = undefined;
977 ip.namespaces_free_list.append(gpa, index) catch {
978 // In order to keep `destroyNamespace` a non-fallible function, we ignore memory
979 // allocation failures here, instead leaking the Namespace until garbage collection.
980 };
981}
982
983const InternPool = @This();
984
985const std = @import("std");
986const Allocator = std.mem.Allocator;
987const Hash = std.hash.Wyhash;
988const assert = std.debug.assert;
989
990const Comp = @import("Comp.zig");
991const Cache = @import("Cache.zig");
992
993const Teg = @import("Teg.zig");
994const Air = @import("Air.zig");
995
996const dod = @import("dod.zig");
997const Float = dod.Float;
998
999const tracer = @import("tracer");
1000
1001test InternPool {
1002 const testing = std.testing;
1003 std.testing.refAllDeclsRecursive(@This());
1004
1005 var ip_struct: InternPool = .{ .gpa = std.testing.allocator };
1006 const ip = &ip_struct;
1007 defer ip.deinit();
1008
1009 const str = try ip.getOrPutString("hello", .no_embedded_nulls);
1010 try testing.expectEqualStrings("hello", str.toSlice(ip));
1011 const str2 = try ip.getOrPutString("hello", .no_embedded_nulls);
1012 try testing.expectEqual(str, str2);
1013
1014 const int1 = try ip.get(.{ .int32 = 21483 });
1015 try testing.expectEqual(@as(i32, 21483), ip.indexToKey(int1).int32);
1016 const int2 = try ip.get(.{ .int32 = 21483 });
1017 try testing.expectEqual(int1, int2);
1018}
src/compiler/JsLink.zig created+457
......@@ -0,0 +1,457 @@
1/// This is the JavaScript backend's linker.
2/// It takes in a compilation, a list of `JsTree`s and creates a JavaScript
3/// bundle out of them.
4comp: *Comp,
5
6/// Value is an index into `link_strings`
7decls: std.AutoHashMapUnmanaged(Decl.Index, LinkString.Index) = .{},
8link_strings: std.ArrayHashMapUnmanaged(
9 LinkString,
10 GlobalOwner,
11 LinkString.Context,
12 true,
13) = .{},
14string_bytes: std.ArrayListUnmanaged(u8) = .{},
15
16/// The first declaration is the function containing top-level code.
17pub fn linkJsCode(
18 comp: *Comp,
19 decls: []const Decl.Index,
20 writer: anytype,
21) !void {
22 log.info("Linking {d} decls", .{decls.len});
23
24 // Arena allocator is used for the main flow
25 var link: JsLink = .{ .comp = comp };
26 defer link.deinit();
27
28 // Temp allocator is used for extremely temporary string work.
29 // var temp_sfa = std.heap.stackFallback(4096, comp.gpa);
30 // const temp_alloc = temp_sfa.get();
31
32 // 1. Resolve all global identifiers. Function names must not shadow external
33 // names. We also need this list prepared so that local identifier names do not
34 // shadow functions or externals.
35 const root_decl = comp.intern_pool.declPtrConst(decls[0]);
36 assert(root_decl.kind == .named);
37 assert(root_decl.name == .empty);
38
39 for (root_decl.codegen.?.extern_symbols) |extern_symbol| {
40 try link.reserveExternSymbol(extern_symbol);
41 }
42
43 for (decls[1..]) |decl_index| {
44 assert(decl_index != decls[0]);
45
46 const decl = comp.intern_pool.declPtrConst(decl_index);
47
48 // todo: add all externs first
49 for (decl.codegen.?.extern_symbols) |extern_symbol| {
50 try link.reserveExternSymbol(extern_symbol);
51 }
52
53 const name = decl.name.toSlice(&comp.intern_pool);
54 assert(name.len > 0);
55
56 const gop = try link.link_strings.getOrPutContextAdapted(
57 comp.gpa,
58 name,
59 LinkString.SliceContext{ .link = &link },
60 .{ .link = &link },
61 );
62 if (gop.found_existing) {
63 @panic("TODO: handle this shadowing case.");
64 }
65 const name_str = try link.appendLinkString(name);
66 gop.key_ptr.* = name_str;
67 gop.value_ptr.* = .decl;
68 try link.decls.putNoClobber(comp.gpa, decl_index, @enumFromInt(gop.index));
69 }
70
71 try writer.print("// Compiler Demo. Do not use in production\n", .{});
72
73 // 2. Print every function decl
74 for (decls[1..]) |decl_index| {
75 try link.printDecl(decl_index, writer);
76 }
77
78 // 3. Print the top-level code
79 try link.printDeclContents(&root_decl.codegen.?, writer);
80}
81
82fn deinit(link: *JsLink) void {
83 defer link.* = undefined;
84 link.decls.deinit(link.comp.gpa);
85 link.link_strings.deinit(link.comp.gpa);
86 link.string_bytes.deinit(link.comp.gpa);
87}
88
89fn reserveExternSymbol(link: *JsLink, extern_symbol: String.WithLen) !void {
90 const name = extern_symbol.toSlice(&link.comp.intern_pool);
91 const gop = try link.link_strings.getOrPutContextAdapted(
92 link.comp.gpa,
93 name,
94 LinkString.SliceContext{ .link = link },
95 .{ .link = link },
96 );
97
98 if (gop.found_existing) {
99 if (gop.value_ptr.* != .external) {
100 @panic("TODO: Rename Decl");
101 }
102 }
103 gop.key_ptr.* = try link.appendLinkString(name);
104 gop.value_ptr.* = .external;
105}
106
107fn appendLinkString(link: *JsLink, string: []const u8) !LinkString {
108 const start = link.string_bytes.items.len;
109 log.info("Append '{s}' -> .{{ .start = {d}, .len = {d} }}", .{ string, start, string.len });
110 try link.string_bytes.appendSlice(link.comp.gpa, string);
111 return .{ .start = @intCast(start), .len = @intCast(string.len) };
112}
113
114fn declIdentifier(link: *const JsLink, decl: Decl.Index) []const u8 {
115 return (link.decls.get(decl) orelse if (std.debug.runtime_safety) {
116 std.debug.panic("Decl {d} ({}) is not present in link decl map", .{
117 @intFromEnum(decl),
118 link.comp.intern_pool.declPtrConst(decl).name.fmt(&link.comp.intern_pool),
119 });
120 } else unreachable).toString(link).slice(link);
121}
122
123fn printDecl(link: *JsLink, decl_index: Decl.Index, writer: anytype) !void {
124 try writer.writeAll("let ");
125 try writer.writeAll(link.declIdentifier(decl_index));
126 try writer.writeAll(" = ");
127
128 const decl = link.comp.intern_pool.declPtrConst(decl_index);
129 const tree = &decl.codegen.?;
130
131 try writer.writeAll("(");
132 for (0..tree.param_count) |id| {
133 try link.printLocal(@intCast(id), writer);
134 if (id != tree.param_count - 1) try writer.writeAll(", ");
135 }
136 try writer.writeAll(") => {\n");
137 try link.printDeclContents(tree, writer);
138 try writer.writeAll("};\n");
139}
140
141fn printDeclContents(link: *JsLink, tree: *const JsTree, writer: anytype) !void {
142 for (tree.mainStatements()) |stmt| {
143 try link.printExpr(tree, stmt, writer);
144 try writer.writeAll(";\n");
145 }
146}
147
148fn printBlockContents(link: *JsLink, tree: *const JsTree, list: JsTree.IndexList, writer: anytype) !void {
149 assert(list.slice(tree).len > 0);
150 for (list.slice(tree)) |stmt| {
151 try link.printExpr(tree, stmt, writer);
152 switch (tree.nodeTag(stmt)) {
153 else => try writer.writeAll(";"),
154 .stmt_block,
155 .stmt_if,
156 => {},
157 }
158 try writer.writeAll("\n");
159 }
160}
161
162fn printBlock(link: *JsLink, tree: *const JsTree, list: JsTree.IndexList, writer: anytype) !void {
163 if (list.len != 1) {
164 try writer.writeAll("{");
165 try link.printBlockContents(tree, list, writer);
166 try writer.writeAll("}");
167 } else {
168 try link.printBlockContents(tree, list, writer);
169 }
170}
171
172fn printExpr(link: *JsLink, tree: *const JsTree, node: Node.Index, writer: anytype) @TypeOf(writer).Error!void {
173 log.info("print node {d}", .{@intFromEnum(node)});
174 switch (tree.nodeTag(node)) {
175 else => |tag| std.debug.panic("TODO: printExpr .{s}", .{@tagName(tag)}),
176 .root => unreachable,
177
178 .plain_value => {
179 const plain_value: JsTree.PlainValue = tree.nodeData(.plain_value, node);
180 switch (plain_value.value.toKey(&link.comp.intern_pool)) {
181 // TODO: minification
182 .int32 => |val| try writer.print("{d}", .{val}),
183 // TODO: correct printing of many value types
184 .float => |val| try writer.print("{d}", .{val}),
185 .simple_value => |val| try writer.writeAll(switch (val) {
186 .true => "true",
187 .false => "false",
188 // TODO: make codegen decide from null and undefined
189 // instead of emitting void nodes.
190 .void => "undefined",
191 }),
192 // TODO: escapes
193 .string => |val| try writer.print("\"{s}\"", .{val}),
194
195 else => unreachable, // this value is not a plain_value
196 }
197 },
198
199 .local => {
200 const ref: JsTree.LocalRef = tree.nodeData(.local, node);
201 try link.printLocal(ref.id, writer);
202 },
203
204 .global => {
205 const ref: JsTree.GlobalRef = tree.nodeData(.global, node);
206 try writer.writeAll(ref.name.toSlice(&link.comp.intern_pool));
207 },
208
209 .decl => {
210 const ref: JsTree.DeclRef = tree.nodeData(.decl, node);
211 try writer.writeAll(link.declIdentifier(ref.index));
212 },
213
214 .stmt_break => {
215 const stmt: JsTree.StmtBreak = tree.nodeData(.stmt_break, node);
216 try writer.writeAll("break");
217 if (true) {
218 try writer.writeAll(" ");
219 try link.printLabelIdentifier(stmt.label, writer);
220 }
221 },
222
223 .stmt_return => {
224 const stmt: JsTree.StmtReturn = tree.nodeData(.stmt_return, node);
225 try writer.writeAll("return");
226 if (stmt.operand.unwrap()) |operand| {
227 try writer.writeAll(" ");
228 try link.printExpr(tree, operand, writer);
229 }
230 },
231
232 .stmt_declare_local => {
233 const stmt: JsTree.StmtDeclareLocal = tree.nodeData(.stmt_declare_local, node);
234 try writer.writeAll("let ");
235 try link.printLocal(stmt.identifier_index, writer);
236 if (stmt.init.unwrap()) |init| {
237 try writer.writeAll(" = ");
238 try link.printExpr(tree, init, writer);
239 }
240 },
241
242 .stmt_if => {
243 const stmt: JsTree.StmtIf = tree.nodeData(.stmt_if, node);
244 const data = stmt.data.get(tree);
245 try writer.writeAll("if (");
246 try link.printExpr(tree, stmt.cond, writer);
247 try writer.writeAll(") ");
248 if (data.flags.yes_label) {
249 try link.printLabelIdentifier(data.flags.block_label, writer);
250 try writer.writeAll(":");
251 }
252 try link.printBlock(tree, data.yes, writer);
253 if (data.no.len > 0) {
254 try writer.writeAll(" else ");
255 if (data.flags.no_label) {
256 try link.printLabelIdentifier(data.flags.block_label, writer);
257 try writer.writeAll(":");
258 }
259 try link.printBlock(tree, data.no, writer);
260 }
261 },
262
263 .stmt_block => {
264 const stmt: JsTree.StmtBlock = tree.nodeData(.stmt_block, node);
265 try link.printLabelIdentifier(stmt.label, writer);
266 try writer.writeAll(": {\n");
267 try link.printBlockContents(tree, stmt.content.get(tree), writer);
268 try writer.writeAll("}");
269 },
270
271 inline .assign,
272 .add,
273 .sub,
274 .mul,
275 .div,
276 .eq,
277 .lt,
278 .gt,
279 => |tag| {
280 const bin: JsTree.Bin = tree.nodeData(tag, node);
281 const op: Operator = @field(Operator, @tagName(tag));
282 try link.printExpr(tree, bin.lhs, writer);
283 try writer.writeAll(" ");
284 try writer.writeAll(op.symbol());
285 try writer.writeAll(" ");
286 try link.printExpr(tree, bin.rhs, writer);
287 },
288
289 inline .negate => |tag| {
290 const un: JsTree.Un = tree.nodeData(tag, node);
291 const op: Operator = @field(Operator, @tagName(tag));
292 try writer.writeAll(op.symbol());
293 try link.printExpr(tree, un.operand, writer);
294 },
295
296 .ternary => {
297 const ternary: JsTree.Ternary = tree.nodeData(.ternary, node);
298 const data = ternary.data.get(tree);
299 try writer.writeAll("(");
300 try link.printExpr(tree, ternary.cond, writer);
301 try writer.writeAll(" ? ");
302 try link.printExpr(tree, data.yes, writer);
303 try writer.writeAll(" : ");
304 try link.printExpr(tree, data.no, writer);
305 try writer.writeAll(")");
306 },
307
308 .call => {
309 const call: JsTree.Call = tree.nodeData(.call, node);
310 const args = call.args.get(tree);
311
312 try link.printExpr(tree, call.base, writer);
313 try writer.writeAll("(");
314
315 for (args.slice(tree), 0..) |arg, i| {
316 if (i != 0) try writer.writeAll(", ");
317 try link.printExpr(tree, arg, writer);
318 }
319
320 try writer.writeAll(")");
321 },
322
323 .field => {
324 const field: JsTree.Field = tree.nodeData(.field, node);
325
326 try link.printExpr(tree, field.base, writer);
327 try writer.writeAll(".");
328 try writer.writeAll(field.field.toSlice(&link.comp.intern_pool));
329 },
330 }
331}
332
333fn printLocal(link: *JsLink, local: JsTree.LocalIndex, writer: anytype) !void {
334 _ = link; // todo: be smarter about this
335 try writer.print("v{d}", .{local});
336}
337
338fn printLabelIdentifier(link: *JsLink, local: JsTree.LocalIndex, writer: anytype) !void {
339 _ = link; // todo: be smarter about this
340 try writer.print("label{d}", .{local});
341}
342
343const GlobalOwner = enum {
344 external,
345 decl,
346};
347
348// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence
349const Level = enum {
350 comma,
351 assignment,
352 conditional,
353 logical_or,
354 logical_and,
355 bitwise_or,
356 bitwise_xor,
357 bitwise_and,
358 equality,
359 relational,
360 bitshift,
361 additive,
362 multiplicative,
363 exponentiation,
364 prefix,
365 postfix,
366 new,
367 access_and_call,
368 member,
369};
370
371const Operator = enum {
372 assign,
373 add,
374 sub,
375 mul,
376 div,
377 eq,
378 lt,
379 gt,
380 negate,
381
382 pub fn symbol(op: Operator) []const u8 {
383 return switch (op) {
384 .assign => "=",
385 .add => "+",
386 .sub => "-",
387 .mul => "*",
388 .div => "/",
389 .eq => "===",
390 .lt => "<",
391 .gt => ">",
392
393 .negate => "-",
394 };
395 }
396};
397
398const LinkString = struct {
399 start: u32,
400 len: u32,
401
402 fn slice(str: LinkString, link: *const JsLink) []const u8 {
403 return link.string_bytes.items[str.start..][0..str.len];
404 }
405
406 const Index = enum(u32) {
407 _,
408
409 pub fn toString(index: Index, link: *const JsLink) LinkString {
410 return link.link_strings.keys()[@intFromEnum(index)];
411 }
412 };
413
414 const Context = struct {
415 link: *const JsLink,
416
417 pub fn hash(ctx: Context, str: LinkString) u32 {
418 // assert(str.start + str.len < ctx.link.string_bytes.items.len);
419
420 return @truncate(Wyhash.hash(0, str.slice(ctx.link)));
421 }
422
423 pub fn eql(ctx: Context, a: LinkString, b: LinkString, _: usize) bool {
424 assert(a.start + a.len < ctx.link.string_bytes.items.len);
425 assert(b.start + b.len < ctx.link.string_bytes.items.len);
426 return a.start == b.start;
427 }
428 };
429
430 const SliceContext = struct {
431 link: *const JsLink,
432
433 pub fn hash(_: SliceContext, str: []const u8) u32 {
434 return @truncate(Wyhash.hash(0, str));
435 }
436
437 pub fn eql(ctx: SliceContext, a: []const u8, b: LinkString, _: usize) bool {
438 // since the given string is not in string_bytes we do comparisons
439 return std.mem.eql(u8, a, b.slice(ctx.link));
440 }
441 };
442};
443
444const JsLink = @This();
445const std = @import("std");
446const assert = std.debug.assert;
447const Wyhash = std.hash.Wyhash;
448const log = std.log.scoped(.js_link);
449
450const Comp = @import("Comp.zig");
451const Decl = Comp.Decl;
452
453const JsTree = @import("JsTree.zig");
454const Node = JsTree.Node;
455
456const InternPool = @import("InternPool.zig");
457const String = InternPool.String;
src/compiler/JsNodeGen.zig created+541
......@@ -0,0 +1,541 @@
1//! Air -> JsTree
2gpa: Allocator,
3arena: Allocator,
4air: *const Air,
5ip: *const InternPool,
6
7nodes: Node.List,
8extra: dod.ExtraStore(.tfr, .@"var"),
9extern_symbols: std.AutoArrayHashMapUnmanaged(InternPool.String.WithLen, void),
10decl_deps: std.AutoArrayHashMapUnmanaged(Decl.Index, void),
11
12param_count: u32,
13air_node_map: []MapEntry,
14scratch: std.ArrayListUnmanaged(u32),
15next_id: u32,
16next_label: u32,
17
18const Error = error{
19 OutOfMemory,
20};
21
22const MapEntry = union {
23 none: void,
24 node: Node.Index,
25 block: packed struct {
26 used: bool = false,
27 index: u31,
28 },
29};
30
31pub fn init(gpa: Allocator, arena: Allocator, ip: *const InternPool, air: *const Air) !JsNodeGen {
32 const air_node_map = try arena.alloc(MapEntry, air.insts.len);
33 if (std.debug.runtime_safety) {
34 @memset(air_node_map, .{ .none = {} });
35 }
36 const param_count = air.paramCount();
37 return .{
38 .gpa = gpa,
39 .arena = arena,
40 .air = air,
41 .ip = ip,
42 .param_count = param_count,
43 .nodes = .{},
44 .extern_symbols = .{},
45 .decl_deps = .{},
46 .air_node_map = air_node_map,
47 .extra = .{},
48 .scratch = .{},
49 .next_id = param_count,
50 .next_label = 0,
51 };
52}
53
54pub fn deinit(c: *JsNodeGen) void {
55 defer c.* = undefined;
56 c.nodes.deinit(c.gpa);
57 c.extra.deinit(c.gpa);
58 c.extern_symbols.deinit(c.gpa);
59 c.decl_deps.deinit(c.gpa);
60
61 // arena can deinit the rest
62}
63
64pub fn finalize(c: *JsNodeGen) !JsTree {
65 // todo: errdefer will probably leave memory in a fully invalid state
66 try c.nodes.setCapacity(c.gpa, c.nodes.len);
67
68 var extra = try c.extra.toOwnedStore(c.gpa);
69 errdefer extra.deinit(c.gpa);
70
71 // todo: i dont like this dupe
72 const extern_symbols = try c.gpa.dupe(InternPool.String.WithLen, c.extern_symbols.keys());
73 errdefer c.gpa.free(extern_symbols);
74
75 const decl_deps = try c.gpa.dupe(Decl.Index, c.decl_deps.keys());
76 errdefer c.gpa.free(decl_deps);
77
78 defer c.deinit();
79
80 return .{
81 .param_count = c.param_count,
82 .nodes = c.nodes.toOwnedSlice(),
83 .extern_symbols = extern_symbols,
84 .decl_deps = decl_deps,
85 .extra = extra,
86 };
87}
88
89pub fn build(c: *JsNodeGen) !void {
90 assert(c.air.instTag(.main_block) == .block);
91 const statements = c.air.instData(.block, .main_block).slice(c.air);
92
93 const scratch_top = c.scratch.items.len;
94 defer c.scratch.shrinkRetainingCapacity(scratch_top);
95
96 _ = try c.add(.root, undefined);
97
98 for (statements[0..c.param_count], 1..) |stmt, expected_id| {
99 assert(@intFromEnum(stmt) == expected_id);
100 assert(c.air.instTag(stmt) == .arg);
101 c.air_node_map[@intFromEnum(stmt)] = .{
102 .node = try c.add(.local, .{ .id = @intFromEnum(stmt) - 1 }),
103 };
104 }
105
106 for (statements[c.param_count..]) |stmt| {
107 try c.codeGenInst(stmt);
108 }
109
110 const items = c.scratch.items[scratch_top..];
111 const top = c.extra.list.items.len;
112 try c.extra.list.appendSlice(c.gpa, items);
113 c.nodes.items(.data)[0] = EncodedData.encode(.root, .{
114 .start = @intCast(top),
115 .len = @intCast(items.len),
116 });
117}
118
119fn codeGenInst(c: *JsNodeGen, inst: Air.Inst.Index) Error!void {
120 std.debug.print("JsNodeGen: #{d}\n", .{@intFromEnum(inst)});
121 switch (c.air.instTag(inst)) {
122 .arg => unreachable, // codegen must skip over these instructions.
123
124 .block => try c.codeGenBlock(inst),
125
126 .js_global_ref => {
127 const data: InternPool.String.WithLen = c.air.instData(.js_global_ref, inst);
128 try c.extern_symbols.put(c.gpa, data, {});
129 c.air_node_map[@intFromEnum(inst)] = .{
130 .node = try c.add(.global, .{ .name = data }),
131 };
132 },
133
134 inline .gt,
135 .lt,
136 .eq,
137 .add,
138 .sub,
139 .mul,
140 .div,
141 => |tag| {
142 const data: Air.Bin = c.air.instData(tag, inst);
143 const node = try c.add(@field(JsTree.Tag, @tagName(tag)), .{
144 .lhs = try c.fetchExpression(data.lhs),
145 .rhs = try c.fetchExpression(data.rhs),
146 });
147 try c.markExpression(inst, node);
148 },
149
150 inline .negate,
151 => |tag| {
152 const data: Air.Un = c.air.instData(tag, inst);
153 const node = try c.add(@field(JsTree.Tag, @tagName(tag)), .{
154 .operand = try c.fetchExpression(data.operand),
155 });
156 try c.markExpression(inst, node);
157 },
158
159 .field => {
160 const data: Air.Field = c.air.instData(.field, inst);
161 const node = try c.add(.field, .{
162 .base = try c.fetchExpression(data.base),
163 .field = data.field,
164 });
165 try c.markExpression(inst, node);
166 },
167
168 .call => {
169 const data: Air.Call = c.air.instData(.call, inst);
170 const call_data = data.call_data.get(c.air);
171
172 try c.extra.list.ensureUnusedCapacity(c.gpa, call_data.args.len);
173 const start = c.extra.list.items.len;
174 c.extra.list.items.len += call_data.args.len;
175
176 const items = c.extra.list.items[start..][0..call_data.args.len];
177
178 for (call_data.args.slice(c.air), items) |arg, *dest| {
179 dest.* = @intFromEnum(try c.fetchExpression(arg));
180 }
181
182 const node = try c.add(.call, .{
183 .base = try c.fetchExpression(data.base),
184 .args = try c.extra.append(c.gpa, JsTree.IndexList, .{
185 .start = @intCast(start),
186 .len = call_data.args.len,
187 }),
188 });
189 try c.markExpression(inst, node);
190 },
191
192 .@"if" => {
193 const data: Air.If = c.air.instData(.@"if", inst);
194 const extra = data.data.get(c.air);
195 var cond = try c.fetchExpression(data.cond);
196
197 if (c.canLowerToExpression(extra.yes) and
198 c.canLowerToExpression(extra.no))
199 {
200 const yes = try c.lowerToExpression(extra.yes);
201 const no = try c.lowerToExpression(extra.yes);
202
203 const ternary = try c.add(.ternary, .{
204 .cond = cond,
205 .data = try c.extra.append(c.gpa, JsTree.Ternary.Data, .{
206 .yes = yes,
207 .no = no,
208 }),
209 });
210
211 try c.markExpression(inst, ternary);
212 } else {
213 const flags = c.air.instFlags(inst);
214 const uses = flags.uses;
215
216 const target = if (uses != .none)
217 try c.addTemporary()
218 else
219 null;
220
221 var yes = try c.codeGenBranch(extra.yes, target);
222 var no = try c.codeGenBranch(extra.no, target);
223 assert(!((yes == null) and (no == null))); // no "void else void" branches
224
225 // when `yes` is void, swap the condition so that the `else` is omitted
226 if (yes == null) {
227 const temp_yes = yes;
228 yes = no;
229 no = temp_yes;
230 cond = try c.add(.bool_not, .{
231 .operand = cond,
232 });
233 }
234
235 const main_expr = try c.add(.stmt_if, .{
236 .cond = cond,
237 .data = try c.extra.append(c.gpa, JsTree.StmtIf.Data, .{
238 .yes = yes.?.list,
239 .no = if (no) |branch|
240 branch.list
241 else
242 .{ .start = 0, .len = 0 },
243 .flags = .{
244 .block_label = @intCast(c.next_label + 1),
245 .yes_label = yes.?.requires_label,
246 .no_label = if (no) |branch| branch.requires_label else false,
247 },
248 }),
249 });
250
251 try c.addStmt(main_expr);
252
253 if (target) |node| {
254 try c.markExpression(inst, node);
255 }
256 }
257 },
258
259 .@"break" => {
260 const brk: Air.Break = c.air.instData(.@"break", inst);
261
262 const operand = try c.fetchExpression(brk.value);
263
264 // break #0 -> return
265 if (brk.block_ref == .main_block) {
266 try c.addStmt(try c.add(.stmt_return, .{
267 .operand = operand.toOptional(),
268 }));
269 } else {
270 const block = &c.air_node_map[@intFromEnum(brk.block_ref)].block;
271
272 const block_insts = c.air.instData(.block, brk.block_ref).slice(c.air);
273 if (block_insts[block_insts.len - 1] != inst) {
274 block.used = true;
275 }
276
277 const assignment: Node.Index.Optional = @enumFromInt(c.scratch.items[block.index]);
278 if (assignment.unwrap()) |target| {
279 try c.addStmt(try c.add(.assign, .{
280 .lhs = target,
281 .rhs = operand,
282 }));
283 }
284
285 try c.addStmt(try c.add(.stmt_break, .{
286 .label = c.scratch.items[block.index + 1],
287 }));
288 }
289 },
290 }
291}
292
293fn markExpression(c: *JsNodeGen, inst: Air.Inst.Index, node: Node.Index) !void {
294 const flags: Air.Flags = c.air.instFlags(inst);
295 const i = @intFromEnum(inst);
296 switch (flags.uses) {
297 .none => {
298 try c.addStmt(node);
299 },
300 .single => {
301 // TODO: this is not aware of side effect fences
302 c.air_node_map[i] = .{ .node = node };
303 },
304 .multi => {
305 const decl = try c.add(.stmt_declare_local, .{
306 .identifier_index = c.next_id,
307 .init = node.toOptional(),
308 });
309 try c.addStmt(decl);
310 const id = try c.add(.local, .{ .id = c.next_id });
311 c.next_id += 1;
312 c.air_node_map[i] = .{ .node = id };
313 },
314 }
315}
316
317fn fetchExpression(c: *JsNodeGen, inst: Air.Inst.Ref) !Node.Index {
318 return switch (inst.unwrap()) {
319 .value => |v| c.valueToExpression(v),
320 .inst => |index| {
321 const data = c.air_node_map[@intFromEnum(index)];
322 if (std.debug.runtime_safety) {
323 const flags: Air.Flags = c.air.instFlags(index);
324 switch (flags.uses) {
325 .none => unreachable, // incorrectly marked instruction as unused,
326 .single => {},
327 .multi => assert(c.nodes.items(.tag)[@intFromEnum(data.node)] == .local),
328 }
329 }
330 return data.node;
331 },
332 };
333}
334
335fn valueToExpression(c: *JsNodeGen, v: Value) !Node.Index {
336 const intern_tag = c.ip.items.items(.tag)[@intFromEnum(v.toIntern())];
337 switch (intern_tag) {
338 .removed => unreachable, // UAF
339
340 .type_function, .simple_type => unreachable, // Type values are not allowed here
341
342 .int32, .float, .string, .simple_value => {
343 return try c.add(.plain_value, .{ .value = v });
344 },
345
346 // funcs always point to decls
347 .func => {
348 const func = v.toKey(c.ip).func;
349 try c.decl_deps.put(c.gpa, func.owner_decl, {});
350 return try c.add(.decl, .{ .index = func.owner_decl });
351 },
352 }
353}
354
355fn addTemporary(c: *JsNodeGen) !Node.Index {
356 try c.addStmt(try c.add(.stmt_declare_local, .{
357 .identifier_index = c.next_id,
358 .init = .none,
359 }));
360 const id = try c.add(.local, .{ .id = c.next_id });
361 c.next_id += 1;
362 return id;
363}
364
365const Branch = struct {
366 requires_label: bool,
367 list: JsTree.IndexList,
368};
369
370fn codeGenBranch(c: *JsNodeGen, inst: Air.Inst.Ref, temporary: ?Node.Index) !?Branch {
371 const scratch_top = c.scratch.items.len;
372 defer c.scratch.shrinkRetainingCapacity(scratch_top);
373
374 c.next_label += 1;
375 defer c.next_label -= 1;
376 const this_label = c.next_label;
377
378 var requires_label = false;
379 var items = switch (inst.unwrap()) {
380 .value => |v| items: {
381 if (v.ip_index == .void_value)
382 return null;
383 if (temporary) |temp| {
384 const node = try c.valueToExpression(v);
385 try c.addStmt(try c.add(.assign, .{
386 .lhs = temp,
387 .rhs = node,
388 }));
389 }
390 break :items c.scratch.items[scratch_top..];
391 },
392 .inst => |i| items: {
393 assert(c.air.instTag(i) == .block);
394 c.air_node_map[@intFromEnum(i)] = .{
395 .block = .{ .index = @intCast(scratch_top) },
396 };
397
398 try c.scratch.appendSlice(c.arena, &.{
399 @intFromEnum(if (temporary) |temp| temp.toOptional() else .none),
400 this_label,
401 });
402
403 std.debug.print("requires_label {}\n", .{requires_label});
404 const block: Air.IndexList = c.air.instData(.block, i);
405 for (block.slice(c.air)) |item| {
406 try c.codeGenInst(item);
407 }
408
409 requires_label = c.air_node_map[@intFromEnum(i)].block.used;
410
411 break :items c.scratch.items[scratch_top + 2 ..];
412 },
413 };
414
415 const last_node = c.scratch.items[c.scratch.items.len - 1];
416 if (c.nodes.items(.tag)[last_node] == .stmt_break and
417 c.nodes.items(.data)[last_node].decode(.stmt_break, .stmt_break).label == this_label)
418 {
419 items.len -= 1;
420 c.nodes.len -= 1;
421 }
422
423 const top = c.extra.list.items.len;
424 try c.extra.list.appendSlice(c.gpa, items);
425 return .{
426 .requires_label = requires_label,
427 .list = .{
428 .start = @intCast(top),
429 .len = @intCast(items.len),
430 },
431 };
432}
433
434fn codeGenBlock(c: *JsNodeGen, inst: Air.Inst.Index) !void {
435 const scratch_top = c.scratch.items.len;
436 errdefer c.scratch.shrinkRetainingCapacity(scratch_top);
437
438 c.next_label += 1;
439 defer c.next_label -= 1;
440
441 const next_label = c.next_label;
442
443 try c.scratch.appendSlice(c.arena, &.{
444 // @intFromEnum(if (temporary) |temp| temp.toOptional() else .none),
445 @intFromEnum(Node.Index.Optional.none),
446 next_label,
447 });
448
449 c.air_node_map[@intFromEnum(inst)] = .{
450 .block = .{ .index = @intCast(scratch_top) },
451 };
452 const block: Air.IndexList = c.air.instData(.block, inst);
453
454 for (block.slice(c.air)) |item| {
455 try c.codeGenInst(item);
456 }
457
458 const items = c.scratch.items[scratch_top + 2 ..];
459 const top = c.extra.list.items.len;
460 try c.extra.list.appendSlice(c.gpa, items);
461 c.scratch.shrinkRetainingCapacity(scratch_top);
462 try c.addStmt(try c.add(.stmt_block, .{
463 .label = c.next_label,
464 .content = try c.extra.append(c.gpa, JsTree.IndexList, .{
465 .start = @intCast(top),
466 .len = @intCast(items.len),
467 }),
468 }));
469}
470
471fn canLowerToExpression(c: *const JsNodeGen, inst: Air.Inst.Ref) bool {
472 return switch (inst.unwrap()) {
473 .value => true,
474 .inst => |index| switch (c.air.instTag(index)) {
475 .js_global_ref,
476 .call,
477 .arg,
478 .field,
479 .add,
480 .sub,
481 .mul,
482 .div,
483 .eq,
484 .lt,
485 .gt,
486 .negate,
487 => true,
488 .@"break",
489 => false,
490 // TODO: these depend
491 .block => false,
492 .@"if" => false,
493 },
494 };
495}
496
497fn lowerToExpression(c: *JsNodeGen, ref: Air.Inst.Ref) !Node.Index {
498 assert(c.canLowerToExpression(ref));
499 return switch (ref.unwrap()) {
500 .value => |v| try c.valueToExpression(v),
501 .inst => |index| {
502 try c.codeGenInst(index);
503 return c.fetchExpression(ref);
504 },
505 };
506}
507
508fn add(c: *JsNodeGen, comptime tag: JsTree.Tag, data: tag.Data()) !Node.Index {
509 try c.nodes.append(c.gpa, .{
510 .tag = tag,
511 .data = EncodedData.encode(tag, data),
512 });
513 return @enumFromInt(c.nodes.len - 1);
514}
515
516fn addStmt(c: *JsNodeGen, stmt: Node.Index) !void {
517 try c.scratch.append(c.arena, @intFromEnum(stmt));
518}
519
520const JsNodeGen = @This();
521
522const std = @import("std");
523const assert = std.debug.assert;
524const Allocator = std.mem.Allocator;
525
526const InternPool = @import("InternPool.zig");
527const NullTerminatedString = InternPool.NullTerminatedString;
528
529const Comp = @import("Comp.zig");
530const Decl = Comp.Decl;
531
532const JsTree = @import("JsTree.zig");
533const EncodedData = JsTree.EncodedData;
534const Node = JsTree.Node;
535const LocalIndex = JsTree.LocalIndex;
536
537const Air = @import("Air.zig");
538
539const Value = @import("Value.zig");
540
541const dod = @import("dod.zig");
src/compiler/JsTree.zig created+245
......@@ -0,0 +1,245 @@
1//! JavaScript Function Representation is the last representation before javascript
2//! source bytes are created. One notable thing here is that all local identifiers
3//! are numbered, instead of being named. The first `param_count` local identifiers
4//! refer to parameters, and the rest refer to the respective block that defined them.
5
6param_count: u32,
7nodes: Node.List.Slice,
8extern_symbols: []String.WithLen,
9decl_deps: []Decl.Index,
10extra: ExtraStore,
11
12pub const LocalIndex = u32;
13pub const ExtraStore = dod.ExtraStore(.tfr, .@"const");
14pub const EncodedData = dod.EncodedData(Tag, u64);
15pub const ExtraIndex = ExtraStore.Index;
16
17pub const Tag = enum {
18 root,
19 /// Only allow simple value types: int, float, string, bool
20 plain_value,
21 /// Reference to a local identifier (give an VarId)
22 local,
23 /// Reference to a global identifier (give a NullTerminatedString)
24 global,
25 /// Reference to a declaration (contains a Decl.Index)
26 decl,
27 // bin ops
28 assign,
29 add,
30 sub,
31 mul,
32 div,
33 eq,
34 lt,
35 gt,
36 // exprs
37 call,
38 field,
39 ternary,
40 // un ops
41 negate,
42 bool_not,
43 // stmts
44 /// `let ... = ...`
45 stmt_declare_local,
46 /// `if (...) ... else ...`
47 stmt_if,
48 /// `break ... ...`
49 stmt_break,
50 /// `return ...` (break from block #0)
51 stmt_return,
52 /// `...: { ... }`
53 stmt_block,
54
55 pub fn Data(comptime tag: Tag) type {
56 return switch (tag) {
57 .root => IndexList,
58 .plain_value => PlainValue,
59 .local => LocalRef,
60 .global => GlobalRef,
61 .decl => DeclRef,
62 .assign,
63 .add,
64 .sub,
65 .mul,
66 .div,
67 .eq,
68 .lt,
69 .gt,
70 => Bin,
71 .ternary => Ternary,
72 .negate,
73 .bool_not,
74 => Un,
75 .call => Call,
76 .field => Field,
77 .stmt_if => StmtIf,
78 .stmt_break => StmtBreak,
79 .stmt_return => StmtReturn,
80 .stmt_declare_local => StmtDeclareLocal,
81 .stmt_block => StmtBlock,
82 };
83 }
84};
85
86pub const Node = struct {
87 tag: Tag,
88 data: EncodedData,
89
90 pub const List = std.MultiArrayList(Node);
91
92 pub const Index = enum(u32) {
93 root = 0,
94 _,
95
96 pub fn toOptional(i: Index) Optional {
97 return @enumFromInt(@intFromEnum(i));
98 }
99
100 pub const Optional = enum(u32) {
101 none = std.math.maxInt(u32),
102 _,
103
104 pub fn unwrap(oi: Optional) ?Index {
105 return if (oi == .none)
106 null
107 else
108 @enumFromInt(@intFromEnum(oi));
109 }
110 };
111 };
112};
113
114pub const PlainValue = struct {
115 value: Value,
116 unused: Unused = .zero,
117};
118
119pub const LocalRef = struct {
120 id: LocalIndex,
121 unused: Unused = .zero,
122};
123
124pub const GlobalRef = struct {
125 name: InternPool.String.WithLen,
126};
127
128pub const DeclRef = struct {
129 index: Decl.Index,
130 unused: Unused = .zero,
131};
132
133pub const StmtDeclareLocal = struct {
134 identifier_index: u32,
135 init: Node.Index.Optional,
136};
137
138pub const Bin = struct {
139 lhs: Node.Index,
140 rhs: Node.Index,
141};
142
143pub const Un = struct {
144 operand: Node.Index,
145 unused: Unused = .zero,
146};
147
148pub const Call = struct {
149 base: Node.Index,
150 args: ExtraIndex(IndexList),
151};
152
153pub const Field = struct {
154 base: Node.Index,
155 field: NullTerminatedString,
156};
157
158pub const IndexList = struct {
159 start: u32,
160 len: u32,
161
162 pub fn slice(list: IndexList, tree: *const JsTree) []const Node.Index {
163 return @ptrCast(tree.extra.list.items[list.start..][0..list.len]);
164 }
165};
166
167pub const StmtIf = struct {
168 cond: Node.Index,
169 data: ExtraIndex(Data),
170
171 pub const Data = struct {
172 yes: IndexList,
173 no: IndexList,
174 flags: Flags,
175 };
176
177 pub const Flags = packed struct(u32) {
178 block_label: u16,
179 yes_label: bool,
180 no_label: bool,
181 _: u14 = 0,
182 };
183};
184
185pub const Ternary = struct {
186 cond: Node.Index,
187 data: ExtraIndex(Data),
188
189 pub const Data = struct {
190 yes: Node.Index,
191 no: Node.Index,
192 };
193};
194
195pub const StmtBreak = struct {
196 label: u32,
197 unused: Unused = .zero,
198};
199
200pub const StmtReturn = struct {
201 operand: Node.Index.Optional,
202 unused: Unused = .zero,
203};
204
205pub const StmtBlock = struct {
206 label: u32,
207 content: ExtraIndex(IndexList),
208};
209
210pub fn deinit(tree: *JsTree, gpa: std.mem.Allocator) void {
211 tree.nodes.deinit(gpa);
212 gpa.free(tree.decl_deps);
213 gpa.free(tree.extern_symbols);
214 tree.extra.deinit(gpa);
215}
216
217pub fn nodeTag(tree: JsTree, index: Node.Index) Tag {
218 return tree.nodes.items(.tag)[@intFromEnum(index)];
219}
220
221pub fn nodeData(tree: JsTree, comptime tag: Tag, index: Node.Index) tag.Data() {
222 return tree.nodes.items(.data)[@intFromEnum(index)].decode(tag, tree.nodeTag(index));
223}
224
225pub fn mainStatements(tree: *const JsTree) []const Node.Index {
226 return tree.nodeData(.root, .root).slice(tree);
227}
228
229const JsTree = @This();
230
231const std = @import("std");
232const assert = std.debug.assert;
233
234const InternPool = @import("InternPool.zig");
235const NullTerminatedString = InternPool.NullTerminatedString;
236const String = InternPool.String;
237
238const Comp = @import("Comp.zig");
239const Decl = Comp.Decl;
240
241const dod = @import("dod.zig");
242
243const Value = @import("Value.zig");
244
245const Unused = @import("Ast.zig").Unused;
src/compiler/Parse.zig created+961
......@@ -0,0 +1,961 @@
1//! Parser takes a Token list and creates a Node list. It does not own the source code or
2//! token data, however transferring into an `Ast` does move token list ownership.
3gpa: Allocator,
4source: []const u8,
5token_tags: []const Token.Tag,
6token_starts: []const ByteOffset,
7
8pos: TokenIndex = 0,
9nodes: Node.List = .{},
10extra: ArrayListUnmanaged(u32) = .{},
11scratch: ArrayListUnmanaged(NodeIndex) = .{},
12
13ast_error: ?AstError = null,
14
15pub fn init(gpa: Allocator, source: []const u8, tokens: Token.List.Slice) Parse {
16 return Parse{
17 .gpa = gpa,
18 .source = source,
19 .token_tags = tokens.items(.tag),
20 .token_starts = tokens.items(.offset),
21 };
22}
23
24/// This transfers the parse result into an Ast
25pub fn finalize(p: *Parse, token_slice: Token.List.Slice) !Ast {
26 const extra = try p.extra.toOwnedSlice(p.gpa);
27 defer p.* = undefined;
28 assert(token_slice.items(.tag).ptr == p.token_tags.ptr);
29 assert(token_slice.items(.offset).ptr == p.token_starts.ptr);
30 p.scratch.deinit(p.gpa);
31
32 const ast: Ast = .{
33 .utf8_source = p.source,
34 .tokens = token_slice,
35 .nodes = p.nodes.toOwnedSlice(),
36 .extra = extra,
37 };
38
39 ast.dumpToWriter(std.io.getStdErr().writer()) catch {};
40
41 return ast;
42}
43
44/// This discards all parser work
45pub fn deinit(p: *Parse) void {
46 defer p.* = undefined;
47 p.nodes.deinit(p.gpa);
48 p.extra.deinit(p.gpa);
49 p.scratch.deinit(p.gpa);
50}
51
52pub const Error = error{
53 SyntaxError,
54};
55
56/// When error.SyntaxError is returned, `p.err` is set to this as well as `p.pos`
57pub const AstError = union(enum) {
58 expected_token: Token.Tag,
59 expected_statement,
60 expected_expr,
61 expected_else_value,
62};
63
64pub fn parseRoot(p: *Parse) !void {
65 const trace = t.trace(@src());
66 defer trace.end();
67
68 assert(p.nodes.len == 0);
69 assert(p.extra.items.len == 0);
70 assert(p.pos == 0);
71 assert(p.nodes.capacity > 0);
72
73 p.nodes.appendAssumeCapacity(.{
74 .tag = .root,
75 .main_token = 0,
76 .data = undefined,
77 });
78
79 const list = try p.parseModuleMembers();
80 p.nodes.items(.data)[0] = .{
81 .list = list,
82 };
83}
84
85fn parseModuleMembers(p: *Parse) !SubList {
86 const trace = t.trace(@src());
87 defer trace.end();
88
89 const scratch_top = p.scratch.items.len;
90 defer p.scratch.shrinkRetainingCapacity(scratch_top);
91
92 // skip module doc comment
93 while (p.eatToken(.module_doc_comment)) |_| {}
94
95 while (true) switch (p.currentTag()) {
96 .kw_let,
97 .kw_var,
98 .kw_fn,
99 .kw_pub,
100 => try p.scratch.append(p.gpa, try p.parseModuleDecl()),
101
102 .kw_struct,
103 .kw_enum,
104 .kw_trait,
105 .kw_impl,
106 => return error.Todo,
107
108 .eof => break,
109
110 else => {
111 if (!p.isExpressionStart()) {
112 return p.fail(.expected_statement);
113 }
114
115 try p.scratch.append(p.gpa, try p.parseExpression());
116 _ = try p.expect(.semi);
117 },
118 };
119
120 return p.scratchToSubList(scratch_top);
121}
122
123fn parseModuleDecl(p: *Parse) !NodeIndex {
124 const trace = t.trace(@src());
125 defer trace.end();
126
127 _ = p.eatToken(.kw_pub);
128
129 return switch (p.currentTag()) {
130 .kw_var, .kw_let => try p.parseVarDecl(),
131
132 .kw_fn => try p.parseFnDecl(),
133
134 else => unreachable,
135 };
136}
137
138fn parseFnDecl(p: *Parse) !NodeIndex {
139 const trace = t.trace(@src());
140 defer trace.end();
141
142 const main_token = p.assertToken(.kw_fn);
143
144 _ = try p.expect(.id);
145
146 _ = try p.expect(.start_paren);
147
148 const params = try p.parseParamList();
149
150 const return_type = if (p.eatToken(.minus_lt)) |_|
151 try p.parseExpression()
152 else
153 null;
154
155 try p.expectWithoutAdvance(.start_brace);
156 const body = try p.parseBlock();
157
158 const extra = try p.addExtra(Ast.FnDecl, .{
159 .params = params,
160 .return_type = optionalNodeIndex(return_type),
161 });
162
163 return try p.addNode(.{
164 .tag = .fn_decl,
165 .main_token = main_token,
166 .data = .{ .fn_decl = .{
167 .body = body,
168 .extra = extra,
169 } },
170 });
171}
172
173fn parseParamList(p: *Parse) !SubList {
174 const trace = t.trace(@src());
175 defer trace.end();
176
177 const scratch_top = p.scratch.items.len;
178 defer p.scratch.shrinkRetainingCapacity(scratch_top);
179
180 while (true) {
181 const trace_param = t.traceFmt(@src(), "Param {d}", .{p.scratch.items.len - scratch_top});
182 defer trace_param.end();
183
184 if (p.currentTag() == .end_paren) break;
185
186 try p.scratch.append(p.gpa, try p.parseParamDecl());
187
188 switch (p.currentTag()) {
189 .comma => {
190 _ = p.nextToken();
191 },
192 .end_paren => break,
193 else => return p.fail(.{ .expected_token = .comma }),
194 }
195 }
196
197 _ = p.assertToken(.end_paren);
198
199 return p.scratchToSubList(scratch_top);
200}
201
202fn parseParamDecl(p: *Parse) !NodeIndex {
203 const trace = t.trace(@src());
204 defer trace.end();
205
206 const main_token = p.assertToken(.id);
207
208 _ = try p.expect(.colon);
209
210 const @"type" = try p.parseExpression();
211
212 return try p.addNode(.{
213 .tag = .param_decl,
214 .main_token = main_token,
215 .data = .{ .param_decl = .{
216 .type = @"type",
217 } },
218 });
219}
220
221fn parseBlock(p: *Parse) !NodeIndex {
222 const trace = t.trace(@src());
223 defer trace.end();
224
225 const main_token = p.assertToken(.start_brace);
226
227 const tag, const list = try p.parseBlockContents();
228 assert(p.lastTag() == .end_brace);
229
230 return try p.addNode(.{
231 .tag = tag,
232 .main_token = main_token,
233 .data = .{
234 .list = list,
235 },
236 });
237}
238
239fn parseLabelExpression(p: *Parse) !NodeIndex {
240 const trace = t.trace(@src());
241 defer trace.end();
242
243 const main_token = p.assertToken(.id);
244 _ = try p.expect(.colon);
245 const expr = try p.parseExpression();
246 return try p.addNode(.{
247 .tag = .label,
248 .main_token = main_token,
249 .data = .{ .label = .{
250 .expr = expr,
251 } },
252 });
253}
254
255fn parseBlockContents(p: *Parse) !struct { Node.Tag, SubList } {
256 const trace = t.trace(@src());
257 defer trace.end();
258
259 assert(p.lastTag() == .start_brace);
260 const scratch_top = p.scratch.items.len;
261 defer p.scratch.shrinkRetainingCapacity(scratch_top);
262
263 const tag: Node.Tag = while (true) switch (p.currentTag()) {
264 .kw_let,
265 .kw_var,
266 => try p.scratch.append(p.gpa, try p.parseVarDecl()),
267
268 .end_brace => break .block_void,
269
270 else => {
271 if (!p.isExpressionStart()) {
272 return p.fail(.expected_statement);
273 }
274
275 const first_tag = p.currentTag();
276
277 try p.scratch.append(p.gpa, try p.parseExpression());
278
279 switch (first_tag) {
280 // some control flow and blocks are allowed to not have a
281 // trailing semicolons. this is a stylistic choice.
282 .kw_if,
283 .start_brace,
284 => {
285 // ending a block here should also count as an implicit return,
286 // even though voiding it with a semicolon is not allowed.
287 if (p.currentTag() == .end_brace) break .block_implicit_return;
288 },
289
290 // TODO: make kw_break and kw_return REQUIRE a semicolon, as
291 // they are noreturn statements, doing implicit break on
292 // them makes no sense
293
294 else => {
295 if (p.eatToken(.semi) == null) {
296 // this must be last statement in block if it isnt,
297 // the p.expect(.end_brace) later will fail.
298 break .block_implicit_return;
299 }
300 },
301 }
302 },
303 };
304
305 _ = try p.expect(.end_brace);
306
307 return .{ tag, try p.scratchToSubList(scratch_top) };
308}
309
310/// VarDecl <- (kw_pub)? (kw_var | kw_let) id (colon expr)? equal expr semi
311fn parseVarDecl(p: *Parse) !NodeIndex {
312 const trace = t.trace(@src());
313 defer trace.end();
314
315 const kind = p.currentTag();
316 assert(kind == .kw_var or kind == .kw_let);
317
318 const main_token = p.nextToken();
319
320 _ = try p.expect(.id);
321
322 const data_type: OptionalNodeIndex = if (p.eatToken(.colon)) |_|
323 optionalNodeIndex(try p.parseExpression())
324 else
325 .none;
326
327 _ = try p.expect(.equal);
328
329 const data_init = try p.parseExpression();
330
331 _ = try p.expect(.semi);
332
333 return try p.addNode(.{
334 .tag = .var_decl,
335 .main_token = main_token,
336 .data = .{ .var_decl = .{
337 .type = data_type,
338 .init = data_init,
339 } },
340 });
341}
342
343fn isExpressionStart(p: *Parse) bool {
344 return switch (p.currentTag()) {
345 .id,
346 .number_int,
347 .number_float,
348 .string,
349 .builtin_id,
350 .kw_return,
351 .kw_break,
352 .kw_if,
353 .start_paren,
354 .start_brace,
355 .bang,
356 .tilde,
357 .minus,
358 => true,
359 else => false,
360 };
361}
362
363const Operator = enum(u8) {
364 add = @intFromEnum(Token.Tag.plus),
365 sub = @intFromEnum(Token.Tag.minus),
366 mul = @intFromEnum(Token.Tag.star),
367 div = @intFromEnum(Token.Tag.slash),
368 rem = @intFromEnum(Token.Tag.percent),
369 pow = @intFromEnum(Token.Tag.star_star),
370 @"and" = @intFromEnum(Token.Tag.and_and),
371 @"or" = @intFromEnum(Token.Tag.pipe_pipe),
372 bit_and = @intFromEnum(Token.Tag.@"and"),
373 bit_or = @intFromEnum(Token.Tag.pipe),
374 bit_xor = @intFromEnum(Token.Tag.caret),
375 shl = @intFromEnum(Token.Tag.lt_lt),
376 shr = @intFromEnum(Token.Tag.gt_gt),
377 eq = @intFromEnum(Token.Tag.equal_equal),
378 ne = @intFromEnum(Token.Tag.bang_equal),
379 lt = @intFromEnum(Token.Tag.lt),
380 le = @intFromEnum(Token.Tag.lt_equal),
381 gt = @intFromEnum(Token.Tag.gt),
382 ge = @intFromEnum(Token.Tag.gt_equal),
383 assign = @intFromEnum(Token.Tag.equal),
384 add_assign = @intFromEnum(Token.Tag.plus_equal),
385 sub_assign = @intFromEnum(Token.Tag.minus_equal),
386 mul_assign = @intFromEnum(Token.Tag.star_equal),
387 div_assign = @intFromEnum(Token.Tag.slash_equal),
388 rem_assign = @intFromEnum(Token.Tag.percent_equal),
389 pow_assign = @intFromEnum(Token.Tag.star_star_equal),
390 and_assign = @intFromEnum(Token.Tag.and_and_equal),
391 or_assign = @intFromEnum(Token.Tag.pipe_pipe_equal),
392 bit_and_assign = @intFromEnum(Token.Tag.and_equal),
393 bit_or_assign = @intFromEnum(Token.Tag.pipe_equal),
394 bit_xor_assign = @intFromEnum(Token.Tag.caret_equal),
395 shl_assign = @intFromEnum(Token.Tag.lt_lt_equal),
396 shr_assign = @intFromEnum(Token.Tag.gt_gt_equal),
397
398 const Direction = enum { ltr, rtl };
399
400 /// Only the binary operator levels are actually used,
401 /// but all are listed for reference.
402 const Level = enum {
403 /// e. e[] e()
404 accessor,
405 /// new
406 new,
407 /// ++e, --e, !e, ~e, +e, -e, typeof, void, delete
408 prefix,
409 /// **
410 exponentiation,
411 /// *, /, %
412 multiplicative,
413 /// +, -
414 additive,
415 /// <<, >>, >>>
416 bitwise_shift,
417 /// <, <=, >, >=
418 relational,
419 /// ==, !=
420 equality,
421 /// &
422 bit_and,
423 /// ^
424 bit_xor,
425 /// |
426 bit_or,
427 /// &&
428 logical_and,
429 /// ||, ??
430 logical_or,
431 /// =, +=, -=, *=, /=, %=, / <<=, >>=, >>>=, &=, ^=, |=
432 assign,
433 };
434
435 pub fn direction(op: Operator) Direction {
436 return switch (op) {
437 .pow => .rtl,
438
439 .assign,
440 .add_assign,
441 .sub_assign,
442 .mul_assign,
443 .div_assign,
444 .rem_assign,
445 .pow_assign,
446 .and_assign,
447 .or_assign,
448 .bit_and_assign,
449 .bit_or_assign,
450 .bit_xor_assign,
451 .shl_assign,
452 .shr_assign,
453 => .rtl,
454
455 else => .ltr,
456 };
457 }
458
459 pub fn level(op: Operator) Level {
460 return switch (op) {
461 .pow => .exponentiation,
462 .mul, .div, .rem => .multiplicative,
463 .add, .sub => .additive,
464 .shl, .shr => .bitwise_shift,
465 .lt, .le, .gt, .ge => .relational,
466 .eq, .ne => .equality,
467 .@"and" => .logical_and,
468 .@"or" => .logical_or,
469 .bit_and => .bit_and,
470 .bit_xor => .bit_xor,
471 .bit_or => .bit_or,
472 .assign,
473 .add_assign,
474 .sub_assign,
475 .mul_assign,
476 .div_assign,
477 .rem_assign,
478 .pow_assign,
479 .and_assign,
480 .or_assign,
481 .bit_and_assign,
482 .bit_or_assign,
483 .bit_xor_assign,
484 .shl_assign,
485 .shr_assign,
486 => .assign,
487 };
488 }
489
490 const lut = brk: {
491 const token_tags = std.enums.values(Token.Tag);
492 const op_tags = std.enums.values(Operator);
493
494 var is_operator_lut: [token_tags.len]bool = .{false} ** token_tags.len;
495 for (op_tags) |op| {
496 is_operator_lut[@intFromEnum(op)] = true;
497 }
498
499 var ast_tag_lut: [token_tags.len]Node.Tag = .{.root} ** token_tags.len;
500 for (op_tags) |op| {
501 ast_tag_lut[@intFromEnum(op)] = @field(Node.Tag, @tagName(op));
502 }
503
504 break :brk struct {
505 const is_operator = is_operator_lut;
506 const ast_tag = ast_tag_lut;
507 };
508 };
509
510 pub fn fromToken(tag: Token.Tag) ?Operator {
511 return if (lut.is_operator[@intFromEnum(tag)])
512 @enumFromInt(@intFromEnum(tag))
513 else
514 null;
515 }
516
517 pub fn astTag(op: Operator) Node.Tag {
518 const tag = lut.ast_tag[@intFromEnum(op)];
519 assert(tag != .root); // invalid operator or lut construction
520 return tag;
521 }
522};
523
524fn parseExpression(p: *Parse) anyerror!NodeIndex {
525 const trace = t.trace(@src());
526 defer trace.end();
527
528 assert(p.isExpressionStart());
529
530 // These unary operators are greedy, so handle them first.
531 switch (p.currentTag()) {
532 .kw_return => return p.parseReturn(),
533 .kw_break => return p.parseBreak(),
534 else => {},
535 }
536
537 var expr = try p.parseExpressionPart();
538
539 while (Operator.fromToken(p.currentTag())) |op| {
540 expr = try p.parseBinarySuffix(expr, op);
541 }
542
543 return expr;
544}
545
546fn parseExpressionPart(p: *Parse) !NodeIndex {
547 const trace = t.trace(@src());
548 defer trace.end();
549
550 var expr = switch (p.currentTag()) {
551 // Literals
552 .id => if (p.nextTag() == .colon)
553 try p.parseLabelExpression()
554 else
555 try p.blankNode(.id),
556 .number_int => try p.blankNode(.number_int),
557 .number_float => try p.blankNode(.number_float),
558 .string => try p.blankNode(.string),
559
560 // Complex expressions
561 .builtin_id => try p.parseBuiltinCall(),
562 .kw_return => try p.parseReturn(),
563 .kw_break => try p.parseBreak(),
564 .kw_if => try p.parseIf(),
565 .start_paren => try p.parseGroup(),
566 .start_brace => try p.parseBlock(),
567
568 // Unary operators
569 inline .bang, .tilde, .minus => |tag| unary: {
570 const main_token = p.nextToken();
571 const expr = try p.parseExpressionPart();
572
573 break :unary try p.addNode(.{
574 .tag = switch (tag) {
575 .bang => .bool_not,
576 .tilde => .bit_not,
577 .minus => .negate,
578 else => comptime unreachable,
579 },
580 .main_token = main_token,
581 .data = .{ .un = .{
582 .expr = expr,
583 } },
584 });
585 },
586
587 else => return p.fail(.expected_expr),
588 };
589
590 // parse field, calls, and index expressions
591 while (true) {
592 const trace2 = t.traceFmt(@src(), "(field/call)", .{});
593 defer trace2.end();
594
595 switch (p.currentTag()) {
596 .dot => {
597 const main_token = p.nextToken();
598 const field = try p.expect(.id);
599
600 expr = try p.addNode(.{
601 .tag = .field,
602 .main_token = main_token,
603 .data = .{ .field = .{
604 .base = expr,
605 .field_token = field,
606 } },
607 });
608 },
609 .start_paren => {
610 const main_token = p.nextToken();
611 const args = try p.parseCallArgs();
612
613 const extra = try p.addExtra(Ast.Call, .{
614 .args = args,
615 });
616
617 expr = try p.addNode(.{
618 .tag = .call,
619 .main_token = main_token,
620 .data = .{ .call = .{
621 .base = expr,
622 .extra = extra,
623 } },
624 });
625 },
626 .start_bracket => {
627 const main_token = p.nextToken();
628 const index = try p.parseExpression();
629
630 expr = try p.addNode(.{
631 .tag = .index,
632 .main_token = main_token,
633 .data = .{ .bin = .{
634 .lhs = expr,
635 .rhs = index,
636 } },
637 });
638 },
639
640 else => break,
641 }
642 }
643
644 return expr;
645}
646
647fn parseBinarySuffix(p: *Parse, lhs: NodeIndex, op: Operator) !NodeIndex {
648 const trace = t.trace(@src());
649 defer trace.end();
650
651 const main_token = p.nextToken();
652 const level = op.level();
653 var rhs = try p.parseExpressionPart();
654
655 while (Operator.fromToken(p.currentTag())) |next_op| {
656 const new_level = next_op.level();
657 const use = switch (op.direction()) {
658 .ltr => @intFromEnum(new_level) < @intFromEnum(level),
659 .rtl => @intFromEnum(new_level) <= @intFromEnum(level),
660 };
661 if (!use) break;
662
663 rhs = try p.parseBinarySuffix(rhs, next_op);
664 }
665
666 return try p.addNode(.{
667 .tag = op.astTag(),
668 .main_token = main_token,
669 .data = .{ .bin = .{
670 .lhs = lhs,
671 .rhs = rhs,
672 } },
673 });
674}
675
676fn parseReturn(p: *Parse) !NodeIndex {
677 const trace = t.trace(@src());
678 defer trace.end();
679
680 const main_token = p.assertToken(.kw_return);
681
682 const expr = if (p.isExpressionStart())
683 try p.parseExpression()
684 else
685 null;
686
687 return try p.addNode(.{
688 .tag = .@"return",
689 .main_token = main_token,
690 .data = .{ .@"return" = .{
691 .expr = optionalNodeIndex(expr),
692 } },
693 });
694}
695
696fn parseBreak(p: *Parse) !NodeIndex {
697 const trace = t.trace(@src());
698 defer trace.end();
699
700 const main_token = p.assertToken(.kw_break);
701
702 const label = if (p.eatToken(.colon)) |_|
703 try p.expect(.id)
704 else
705 null;
706
707 const expr = if (p.isExpressionStart())
708 try p.parseExpression()
709 else
710 null;
711
712 return try p.addNode(.{
713 .tag = .@"break",
714 .main_token = main_token,
715 .data = .{ .@"break" = .{
716 .label = optionalTokenIndex(label),
717 .value = optionalNodeIndex(expr),
718 } },
719 });
720}
721
722fn parseBuiltinCall(p: *Parse) !NodeIndex {
723 const trace = t.trace(@src());
724 defer trace.end();
725
726 const main_token = p.assertToken(.builtin_id);
727
728 _ = try p.expect(.start_paren);
729
730 const args = try p.parseCallArgs();
731
732 return try p.addNode(.{
733 .tag = .builtin_call,
734 .main_token = main_token,
735 .data = .{ .list = args },
736 });
737}
738
739fn parseCallArgs(p: *Parse) anyerror!SubList {
740 const trace = t.trace(@src());
741 defer trace.end();
742
743 const scratch_top = p.scratch.items.len;
744 defer p.scratch.shrinkRetainingCapacity(scratch_top);
745
746 while (p.isExpressionStart()) {
747 try p.scratch.append(p.gpa, try p.parseExpression());
748
749 switch (p.currentTag()) {
750 .end_paren => {
751 break;
752 },
753 .comma => {
754 _ = p.nextToken();
755 if (p.currentTag() == .end_paren)
756 break;
757 },
758 else => return p.fail(.{ .expected_token = .end_paren }),
759 }
760 }
761
762 _ = try p.expect(.end_paren);
763
764 return p.scratchToSubList(scratch_top);
765}
766
767fn parseIf(p: *Parse) !NodeIndex {
768 const trace = t.trace(@src());
769 defer trace.end();
770
771 const main_token = p.assertToken(.kw_if);
772
773 const cond = try p.parseExpression();
774
775 _ = try p.expectWithoutAdvance(.start_brace);
776 const yes = try p.parseBlock();
777
778 const no = if (p.eatToken(.kw_else)) |_|
779 try p.parseElseValue()
780 else
781 null;
782
783 const extra = try p.addExtra(Ast.If, .{
784 .yes = yes,
785 .no = optionalNodeIndex(no),
786 });
787
788 return try p.addNode(.{
789 .tag = .@"if",
790 .main_token = main_token,
791 .data = .{ .@"if" = .{
792 .cond = cond,
793 .extra = extra,
794 } },
795 });
796}
797
798fn parseElseValue(p: *Parse) anyerror!NodeIndex {
799 const trace = t.trace(@src());
800 defer trace.end();
801
802 assert(p.lastTag() == .kw_else);
803
804 return switch (p.currentTag()) {
805 .start_brace => p.parseBlock(),
806 .kw_if => p.parseIf(),
807 .id => p.parseLabelExpression(),
808
809 else => return p.fail(.expected_else_value),
810 };
811}
812
813fn parseGroup(p: *Parse) !NodeIndex {
814 const trace = t.trace(@src());
815 defer trace.end();
816
817 const main_token = p.assertToken(.start_paren);
818
819 const expr = try p.parseExpression();
820
821 _ = try p.expect(.end_paren);
822
823 return try p.addNode(.{
824 .tag = .group,
825 .main_token = main_token,
826 .data = .{ .un = .{
827 .expr = expr,
828 } },
829 });
830}
831
832// helpers
833
834/// Append a node and return it's index.
835fn addNode(p: *Parse, node: Node) !NodeIndex {
836 const trace = t.trace(@src());
837 defer trace.end();
838
839 try p.nodes.append(p.gpa, node);
840 // @intCast is always safe because 2^32 tokens alwayss produce less than
841 // 2^32 nodes, and there is a 2^32 limit on source file bytes.
842 return @intCast(p.nodes.len - 1);
843}
844
845fn addExtra(p: *Parse, comptime T: type, extra: T) !ExtraIndex {
846 const trace = t.trace(@src());
847 defer trace.end();
848
849 const extra_top = p.extra.items.len;
850 const bytes = std.mem.asBytes(&extra);
851 try p.extra.appendSlice(p.gpa, std.mem.bytesAsSlice(u32, bytes));
852 return @intCast(extra_top);
853}
854
855fn fail(p: *Parse, ast_error: AstError) Error {
856 p.ast_error = ast_error;
857 std.debug.print("got {s}", .{@tagName(p.currentTag())});
858 return Error.SyntaxError;
859}
860
861fn blankNode(p: *Parse, comptime tag: Token.Tag) !NodeIndex {
862 const main_token = p.assertToken(tag);
863 return try p.addNode(.{
864 .tag = @field(Node.Tag, @tagName(tag)),
865 .main_token = main_token,
866 .data = .{ .none = .{} },
867 });
868}
869
870fn currentTag(p: *Parse) Token.Tag {
871 return p.token_tags[p.pos];
872}
873
874fn lastTag(p: *Parse) Token.Tag {
875 return p.token_tags[p.pos - 1];
876}
877
878fn nextTag(p: *Parse) Token.Tag {
879 return p.token_tags[p.pos + 1];
880}
881
882fn eatToken(p: *Parse, tag: Token.Tag) ?TokenIndex {
883 return if (p.token_tags[p.pos] == tag) p.nextToken() else null;
884}
885
886fn expect(p: *Parse, tag: Token.Tag) !TokenIndex {
887 if (p.currentTag() != tag) {
888 return p.fail(.{ .expected_token = tag });
889 }
890 return p.nextToken();
891}
892
893fn expectWithoutAdvance(p: *Parse, tag: Token.Tag) !void {
894 if (p.currentTag() != tag) {
895 return p.fail(.{ .expected_token = tag });
896 }
897}
898
899fn assertToken(p: *Parse, tag: Token.Tag) TokenIndex {
900 const token = p.nextToken();
901 assert(p.token_tags[token] == tag);
902 return token;
903}
904
905fn nextToken(p: *Parse) TokenIndex {
906 const result = p.pos;
907 p.pos += 1;
908 return result;
909}
910
911fn tokensOnSameLine(p: *Parse, token1: TokenIndex, token2: TokenIndex) bool {
912 return std.mem.indexOfScalar(u8, p.source[p.token_starts[token1]..p.token_starts[token2]], '\n') == null;
913}
914
915fn fullToken(p: *Parse, token: TokenIndex) Token {
916 return Token{
917 .tag = p.token_tags[token],
918 .offset = p.token_starts[token],
919 };
920}
921
922fn scratchToSubList(p: *Parse, scratch_top: usize) !SubList {
923 const trace = t.trace(@src());
924 defer trace.end();
925
926 const items = p.scratch.items[scratch_top..];
927 const extra_top = p.extra.items.len;
928 try p.extra.appendSlice(p.gpa, p.scratch.items[scratch_top..]);
929 return .{
930 .start = @intCast(extra_top),
931 .len = @intCast(items.len),
932 };
933}
934
935const Parse = @This();
936
937const std = @import("std");
938const assert = std.debug.assert;
939const mem = std.mem;
940const Allocator = mem.Allocator;
941const ArrayListUnmanaged = std.ArrayListUnmanaged;
942
943const Tokenizer = @import("Tokenizer.zig");
944const Token = Tokenizer.Token;
945const ByteOffset = Tokenizer.ByteOffset;
946
947const Ast = @import("Ast.zig");
948const Node = Ast.Node;
949const Data = Node.Data;
950const SubList = Ast.SubList;
951const NodeIndex = Ast.NodeIndex;
952const TokenIndex = Ast.TokenIndex;
953const ExtraIndex = Ast.ExtraIndex;
954const OptionalNodeIndex = Ast.OptionalNodeIndex;
955const OptionalTokenIndex = Ast.OptionalTokenIndex;
956const OptionalExtraIndex = Ast.OptionalExtraIndex;
957const optionalTokenIndex = OptionalTokenIndex.init;
958const optionalNodeIndex = OptionalNodeIndex.init;
959const optionalExtraIndex = OptionalExtraIndex.init;
960
961const t = @import("tracer");
src/compiler/Report.zig created+38
......@@ -0,0 +1,38 @@
1errors: []Item = &.{},
2warns: []Item = &.{},
3string_bytes: []u8 = &.{},
4
5pub const Item = struct {
6 offset: ByteOffset,
7 message: String,
8};
9
10pub const String = struct {
11 start: u32,
12 len: u32,
13
14 pub fn message(item: Item, report: Report) []const u8 {
15 return report.string_bytes[item.message_start..][0..item.message_len];
16 }
17};
18
19pub fn hasErrors(report: Report) bool {
20 return report.errors.len > 0;
21}
22
23pub fn hasWarns(report: Report) bool {
24 return report.warns.len > 0;
25}
26
27pub const Wip = struct {
28 errors: ArrayListUnmanaged(Item),
29 warns: ArrayListUnmanaged(Item),
30 string_bytes: ArrayListUnmanaged(u8),
31};
32
33const Report = @This();
34const std = @import("std");
35const ArrayListUnmanaged = std.ArrayListUnmanaged;
36const assert = std.debug.assert;
37pub const Tokenizer = @import("Tokenizer.zig");
38pub const ByteOffset = Tokenizer.ByteOffset;
src/compiler/Sema.zig created+693
......@@ -0,0 +1,693 @@
1/// absolutely stealing zig on this one haha
2/// This struct is created per declaration and chills
3
4// inputs:
5comp: *Comp,
6gpa: Allocator,
7arena: Allocator,
8code: Teg,
9ip: *InternPool,
10decl: *Decl,
11decl_index: Decl.Index,
12namespace_index: Namespace.Index,
13
14// temp data:
15inst_map: InstMap = undefined,
16/// scratch memory. can be used in any place you might temporarily allocate an arraylist
17/// this is most notably used to store block contents before the block is finalized
18scratch: std.ArrayListUnmanaged(u32) = .{},
19params: []InternPool.Index = undefined,
20
21// outputs
22air_instructions: std.MultiArrayList(Air.Inst) = .{},
23air_extra: dod.ExtraStore(.air, .@"var") = .{},
24
25pub const Error = error{
26 /// Analysis has failed, see error list
27 AnalysisFail,
28 /// This analysis is waiting on other dependencies to finish
29 Interrupt,
30};
31
32/// Instead of using a hash map for this, just pre-allocate a list long enough to hold everything.
33/// This is fine because all instructions are going to be adacent in a single declaration.
34const InstMap = struct {
35 items: []Air.Inst.Ref.Optional,
36 start: Teg.Inst.Index,
37
38 pub fn init(allocator: Allocator, code: *const Teg, inst_start: Teg.Inst.Index) !InstMap {
39 const decl = code.declData(inst_start);
40 log.info("pre-allocating {d} map-indicies for InstMap", .{decl.instruction_count});
41 const insts = try allocator.alloc(Air.Inst.Ref.Optional, decl.instruction_count);
42 @memset(insts, .none);
43 return .{
44 .items = insts,
45 .start = inst_start,
46 };
47 }
48
49 pub fn deinit(map: *InstMap, allocator: Allocator) void {
50 allocator.free(map.items);
51 map.* = undefined;
52 }
53
54 pub fn get(map: *InstMap, teg_code: Teg.Inst.Index) ?Air.Inst.Ref {
55 const source_i = @intFromEnum(teg_code);
56 const start_i = @intFromEnum(map.start);
57 assert(source_i >= start_i and source_i <= (start_i + map.items.len));
58 return map.items[source_i - start_i].unwrap();
59 }
60
61 pub fn add(map: *InstMap, from: Teg.Inst.Index, to: Air.Inst.Ref) !void {
62 assert(map.get(from) == null);
63 const source_i = @intFromEnum(from);
64 const start_i = @intFromEnum(map.start);
65 map.items[source_i - start_i] = to.toOptional();
66 }
67};
68
69pub fn init(s: *Sema) !void {
70 const t = tracer.trace(@src());
71 defer t.end();
72
73 try s.comp.current_compile_decls.append(s.gpa, s.decl_index);
74
75 log.info("init sema on index {}", .{s.decl_index});
76 s.inst_map = try InstMap.init(
77 s.arena,
78 &s.code,
79 s.ip.declPtr(s.decl_index).teg_decl_index.unwrap().?.resolve(s.ip),
80 );
81}
82
83pub fn deinit(s: *Sema) void {
84 const t = tracer.trace(@src());
85 defer t.end();
86
87 defer s.* = undefined;
88 s.inst_map.deinit(s.arena);
89 s.scratch.deinit(s.arena);
90 s.air_extra.deinit(s.gpa);
91 s.air_instructions.deinit(s.gpa);
92}
93
94pub fn finalize(s: *Sema) !void {
95 const air: Air = s.tempAir();
96
97 std.debug.print("=" ** 20, .{});
98 std.debug.print("\nAir for @ {d}\n", .{@intFromEnum(s.decl.teg_decl_index)});
99 air.dumpToWriter(s.ip, std.io.getStdErr().writer()) catch {};
100
101 try s.comp.codeGenDecl(s.decl, &air);
102}
103
104// --- analyze functions ---
105
106/// This is the entrypoint to semantic analysis: It all starts at a declaration.
107pub fn analyzeDeclaration(s: *Sema) anyerror!Value {
108 const t = tracer.trace(@src());
109 defer t.end();
110
111 // TODO: turn this entire function into a nonrecursive state machine, so
112 // that analysis can pause and continue at any point. for example, when a
113 // dependency on another declaration is required, but that declaration has
114 // not been resolved yet. or, we are waiting on a file read to happen
115
116 const decl, const fn_decl = s.code.fullFuncDecl(s.inst_map.start);
117 assert(decl.kind == .function);
118
119 s.params = try s.gpa.alloc(InternPool.Index, fn_decl.params.len);
120 defer s.gpa.free(s.params);
121
122 for (fn_decl.params.slice(&s.code), 0..) |param, i| {
123 s.params[i] = (try s.resolveRefComptime(param)).toIntern();
124 }
125
126 const return_type = try s.resolveRefComptime(fn_decl.return_type);
127
128 const root = (try s.analyzeBlock(
129 fn_decl.body.toIndex() orelse
130 @panic("TODO: please change this to Index. Ref is not allowed here anymore"),
131 )).unwrap().inst;
132 assert(@intFromEnum(root) == 0);
133
134 const func_prototype = try s.ip.get(s.gpa, .{
135 .type_function = .{
136 .param_types = s.params,
137 .return_type = return_type.toIntern(),
138 },
139 });
140 const func_value = try s.ip.get(s.gpa, .{
141 .func = .{
142 .ty = func_prototype,
143 .owner_decl = s.decl_index,
144 .teg_body_inst = s.decl.teg_decl_index.unwrap().?,
145 },
146 });
147 const value = Value.fromInterned(func_value);
148 s.decl.val = value;
149 return value;
150}
151
152pub fn analyzeBlock(s: *Sema, in: Teg.Inst.Index) anyerror!Air.Inst.Ref {
153 const t = tracer.trace(@src());
154 defer t.end();
155
156 const block = try s.addBlock();
157
158 try s.inst_map.add(in, block.block_index.toRef());
159
160 const scratch_top = s.scratch.items.len;
161 defer s.scratch.shrinkRetainingCapacity(scratch_top);
162
163 var last: Air.Inst.Ref = undefined;
164 const instructions = s.code.blockInstructions(in);
165 assert(instructions.len != 0);
166 for (instructions) |inst| {
167 last = try s.analyzeCode(inst);
168 }
169
170 if (scratch_top + 1 == s.scratch.items.len) {
171 const single_instruction = s.air_instructions.items(.tag)[@intFromEnum(block.block_index) + 1];
172 if (single_instruction == .@"break") {
173 const data: Air.Break = s.air_instructions.items(.data)[@intFromEnum(block.block_index) + 1]
174 .decode(.@"break", single_instruction);
175 if (data.block_ref == block.block_index) {
176 // This is a block that resolved to just a single value ref with
177 // no side effects. This means we can just lower this to the ref
178 // itself, and give codegen an easier time.
179
180 // Delete the block and the single instruction
181 // Break does not use any extra data, so this call removes both items
182 s.air_instructions.shrinkRetainingCapacity(@intFromEnum(block.block_index));
183
184 return data.value;
185 }
186 }
187 }
188
189 return (try block.finish(s, scratch_top)).toRef();
190}
191
192fn analyzeCode(s: *Sema, in: Teg.Inst.Index) !Air.Inst.Ref {
193 const t = tracer.traceFmt(@src(), "%{d}, {s}", .{ @intFromEnum(in), @tagName(s.code.instTag(in)) });
194 defer t.end();
195
196 log.info("runtime %{d} = .{s}", .{ @intFromEnum(in), @tagName(s.code.instTag(in)) });
197
198 const out = try s.analyzeCodeInner(in);
199 if (s.code.instTag(in) != .block)
200 try s.inst_map.add(in, out);
201
202 return out;
203}
204
205fn analyzeCodeInner(s: *Sema, in: Teg.Inst.Index) !Air.Inst.Ref {
206 switch (s.code.instTag(in)) {
207 .string => {
208 const string: Teg.SlicedString = s.code.instPayload(.string, in);
209 return s.makeValueAir(.{ .string = string.slice(s.code.string_bytes) });
210 },
211 .number => {
212 const number: dod.Float = s.code.instPayload(.number, in);
213 return s.makeValueAir(.{ .float = number.decode() });
214 },
215 .js => {
216 const operand = try s.resolveRefComptime(s.code.instData(.js, in).operand);
217 try s.validateType(operand.type(s.ip), .string_type);
218 return s.addInstRef(.js_global_ref, operand.asString(s.ip));
219 },
220 .field => {
221 const field: Teg.Inst.Bin = s.code.instPayload(.field, in);
222 const base = try s.resolveRefRuntime(field.lhs);
223 const rhs = try s.resolveRefComptime(field.rhs);
224
225 try s.validateType(rhs.type(s.ip), .string_type);
226 const string = rhs.asString(s.ip);
227
228 const base_type = s.typeOfRef(base);
229 log.info("process field with type {any} . {s}", .{ base_type, string.toSlice(s.ip) });
230
231 switch (base_type) {
232 .opaque_type => {
233 return s.addInstRef(.field, .{
234 .base = base,
235 .field = try string.tryToNullTerminatedString(s.ip, s.gpa),
236 });
237 },
238 else => |tag| std.debug.panic("TODO: field analyze with base .{s}", .{@tagName(tag)}),
239 }
240 },
241 .load_decl => {
242 const request: Teg.Inst.StrNode = s.code.instData(.load_decl, in);
243 const decl_name = try s.ip.getOrPutString(
244 s.gpa,
245 request.string.slice(s.code.string_bytes),
246 .no_embedded_nulls,
247 );
248 const decl_index = s.lookupDecl(decl_name) orelse {
249 std.debug.print("did not find {s}\n", .{decl_name.toSlice(s.ip)});
250 return error.AnalysisFail;
251 };
252 const resolved = try s.resolveDecl(decl_index);
253 return resolved.toIntern().toAir();
254 },
255 .param => {
256 const param: Teg.Inst.DataNode = s.code.instData(.param, in);
257 return s.addInstRef(.arg, .{
258 .type = Value.fromInterned(s.params[param.data]),
259 .arg_number = param.data,
260 });
261 },
262 inline .add,
263 .sub,
264 .mul,
265 .div,
266 .lt,
267 .gt,
268 .eq,
269 => |tag| {
270 const bin: Teg.Inst.Bin = s.code.instPayload(tag, in);
271
272 const lhs = try s.resolveRefRuntime(bin.lhs);
273 const rhs = try s.resolveRefRuntime(bin.rhs);
274
275 if (lhs.kind == .value and rhs.kind == .value) {
276 @panic("TODO: comptime math operation");
277 }
278
279 const left_type = s.typeOfRef(lhs);
280 const right_type = s.typeOfRef(rhs);
281
282 // TODO: validate type for operation
283
284 log.info("runtime {s} operation on {} and {}", .{
285 @tagName(tag),
286 left_type.toKey(s.ip),
287 right_type.toKey(s.ip),
288 });
289
290 // todo: type check
291
292 return s.addInstRef(tag.airTag(), .{
293 .lhs = lhs,
294 .rhs = rhs,
295 });
296 },
297 inline .negate => |tag| {
298 const un: Teg.Inst.RefNode = s.code.instData(tag, in);
299
300 const operand = try s.resolveRefRuntime(un.operand);
301
302 const operand_type = s.typeOfRef(operand);
303
304 // TODO: validate type for operation
305
306 log.info("runtime {s} operation on {}", .{
307 @tagName(tag),
308 operand_type.toKey(s.ip),
309 });
310
311 return s.addInstRef(tag.airTag(), .{
312 .operand = operand,
313 });
314 },
315 .@"break" => {
316 const brk: Teg.Inst.Break = s.code.instPayload(.@"break", in);
317 const operand = try s.resolveRefRuntime(brk.operand);
318 const block_ref = s.inst_map.get(brk.block).?.unwrap().inst; // block ref is resolved by teg
319 return s.addInstRef(.@"break", .{
320 .block_ref = block_ref,
321 .value = operand,
322 });
323 },
324 .call => {
325 const call: Teg.Inst.Call = s.code.instPayload(.call, in);
326 const base = try s.resolveRefRuntime(call.base);
327 const base_type = s.typeOfRef(base).toKey(s.ip);
328
329 const is_opaque = base_type == .simple_type and base_type.simple_type == .@"opaque";
330
331 const scratch = try s.arena.alloc(Air.Inst.Ref, call.args.len);
332 defer s.arena.free(scratch);
333 if (!is_opaque) {
334 if (base_type != .type_function) {
335 std.debug.print("Attempt to call non-function {}", .{base_type});
336 return error.AnalysisFail;
337 }
338
339 const proto = base_type.type_function;
340 if (proto.param_types.len != call.args.len) {
341 std.debug.print("Incorrect args. Expect {d}, Got {d}", .{
342 proto.param_types.len,
343 call.args.len,
344 });
345 return error.AnalysisFail;
346 }
347
348 // validate function args
349 for (proto.param_types, call.args.slice(&s.code), 0..) |param, arg, i| {
350 const ref = try s.resolveRefRuntime(arg);
351 scratch[i] = ref;
352 try s.validateType(
353 Value.fromInterned(s.typeOfRef(ref)),
354 param,
355 );
356 }
357 } else {
358 // just copy the args in without checking
359 for (call.args.slice(&s.code), 0..) |arg, i| {
360 const ref = try s.resolveRefRuntime(arg);
361 scratch[i] = ref;
362 }
363 }
364
365 const start = s.air_extra.list.items.len;
366 try s.air_extra.list.appendSlice(s.gpa, @ptrCast(scratch));
367
368 return s.addInstRef(.call, .{
369 .base = base,
370 .call_data = try s.air_extra.append(s.gpa, Air.Call.Data, .{
371 .args = .{
372 .start = @intCast(start),
373 .len = @intCast(scratch.len),
374 },
375 }),
376 });
377 },
378 .ensure_void => {
379 const operand = try s.resolveRefRuntimeExtra(s.code.instData(.ensure_void, in).operand, false);
380 const t = Value.fromInterned(s.typeOfRef(operand));
381 if (!t.isEnsureVoid(s.ip)) {
382 std.debug.print("oh no, we are discarding {} on accident\n", .{
383 t.fmt(s.ip),
384 });
385 }
386 return InternPool.Index.void_value.toAir();
387 },
388 .@"if" => {
389 const data: Teg.Inst.If = s.code.instData(.@"if", in).extra_index.get(&s.code);
390
391 const cond = try s.resolveRefRuntime(data.cond);
392
393 try s.validateType(Value.fromInterned(s.typeOfRef(cond)), .bool_type);
394
395 // If known at comptime, only analyze the taken path
396 if (cond.kind == .value) {
397 @panic("TODO: comptime IF");
398 }
399
400 // Emit a runtime branch
401 const yes_block = try s.analyzeRefToBlock(data.yes);
402 const no_block = try s.analyzeRefToBlock(data.no);
403
404 return try s.addInstRef(.@"if", .{
405 .cond = cond,
406 .data = try s.air_extra.append(s.gpa, Air.If.Data, .{
407 .yes = yes_block,
408 .no = no_block,
409 .union_ty = s.unionType(Value.fromInterned(s.typeOfRef(yes_block)), Value.fromInterned(s.typeOfRef(no_block))),
410 }),
411 });
412 },
413 .block => {
414 const block = try s.analyzeBlock(in);
415 switch (block.unwrap()) {
416 .inst => |t| try s.scratch.append(s.gpa, @intFromEnum(t)),
417 else => {},
418 }
419 return block;
420 },
421 else => |tag| std.debug.panic("TODO: analyze of .{s}", .{@tagName(tag)}),
422 }
423}
424
425fn markInst(s: *Sema, inst: Air.Inst.Index) void {
426 const flags: *Air.Flags = &s.air_instructions.items(.flags)[@intFromEnum(inst)];
427 flags.uses = flags.uses.increment();
428 log.info("markInst {}", .{inst});
429}
430
431fn analyzeRefToBlock(s: *Sema, in: Teg.Inst.Ref) anyerror!Air.Inst.Ref {
432 const t = tracer.trace(@src());
433 defer t.end();
434
435 const scratch_top = s.scratch.items.len;
436 defer s.scratch.shrinkRetainingCapacity(scratch_top);
437
438 const visited = try s.resolveRefRuntime(in);
439
440 if (scratch_top == s.scratch.items.len) {
441 return visited;
442 }
443
444 if (visited.kind == .inst and
445 s.air_instructions.items(.tag)[@intFromEnum(visited.unwrap().inst)] == .block)
446 {
447 return visited;
448 }
449
450 const block = try s.addBlock();
451 return (try block.finish(s, scratch_top)).toRef();
452}
453
454fn resolveRefRuntimeExtra(s: *Sema, ref: Teg.Inst.Ref, mark: bool) anyerror!Air.Inst.Ref {
455 const index = ref.toIndex() orelse {
456 return ref.toInterned().?.toAir();
457 };
458
459 if (s.inst_map.get(index)) |air| {
460 if (mark) {
461 switch (air.unwrap()) {
462 .inst => |i| s.markInst(i),
463 else => {},
464 }
465 }
466 return air;
467 }
468
469 return s.analyzeCode(index);
470}
471
472fn resolveRefRuntime(s: *Sema, ref: Teg.Inst.Ref) anyerror!Air.Inst.Ref {
473 return resolveRefRuntimeExtra(s, ref, true);
474}
475
476fn resolveRefComptime(s: *Sema, ref: Teg.Inst.Ref) !Value {
477 const in = ref.toIndex() orelse {
478 return Value{ .ip_index = ref.toInterned().? };
479 };
480
481 if (s.inst_map.get(in)) |air| {
482 switch (air.unwrap()) {
483 .value => |value| return value,
484 else => {},
485 }
486 }
487
488 const tag = s.code.instTag(in);
489 log.info("comptime %{d} = .{s}", .{ @intFromEnum(in), @tagName(tag) });
490 switch (tag) {
491 .string => {
492 const string: Teg.SlicedString = s.code.instPayload(.string, in);
493 return s.makeValue(.{ .string = string.slice(s.code.string_bytes) });
494 },
495
496 else => std.debug.panic("TODO: comptime analyze of .{s}", .{@tagName(tag)}),
497 }
498}
499
500fn resolveDecl(s: *Sema, decl_index: Decl.Index) !Value {
501 const decl = s.ip.declPtr(decl_index);
502 switch (decl.analysis) {
503 .unreferenced => {
504 decl.analysis = .in_progress;
505
506 // TODO: make this short-circuit the current analysis, move this analyze
507 // job into a "deferred" list, and then make this thread analyze something else
508
509 var arena = std.heap.ArenaAllocator.init(s.gpa);
510 defer arena.deinit();
511
512 var sema: Sema = .{
513 .comp = s.comp,
514 .gpa = s.gpa,
515 .ip = s.ip,
516 .arena = arena.allocator(),
517 .code = try s.ip.namespacePtr(decl.src_namespace).file_scope.getTeg(s.gpa),
518 .decl = decl,
519 .decl_index = decl_index,
520 .namespace_index = decl.src_namespace,
521 };
522
523 try sema.init();
524 defer sema.deinit();
525 const value = try sema.analyzeDeclaration();
526 try sema.finalize();
527
528 decl.analysis = .complete;
529
530 return value;
531 },
532 .complete => {
533 return decl.val;
534 },
535 else => |tag| std.debug.panic("TODO: load_decl when decl is in state .{s}", .{@tagName(tag)}),
536 }
537}
538
539fn unionType(s: *Sema, a: Value, b: Value) Value {
540 if (std.debug.runtime_safety) {
541 assert(a.type(s.ip).toIntern() == .type_type);
542 assert(b.type(s.ip).toIntern() == .type_type);
543 }
544
545 if (a.ip_index == b.ip_index) {
546 return a;
547 }
548
549 if (a.ip_index == .noreturn_type)
550 return b;
551
552 if (b.ip_index == .noreturn_type)
553 return a;
554
555 std.debug.panic(
556 "TODO: when two non-equal types implicitly merge (if/match), this should generate a union type.\na: {}\nb: {}",
557 .{
558 a.fmt(s.ip),
559 b.fmt(s.ip),
560 },
561 );
562 // return s.ip.get(s.gpa, .{ .@"union" = &.{ a, b } });
563}
564
565/// --- helper ---
566const IncompleteBlock = struct {
567 block_index: Air.Inst.Index,
568
569 pub fn finish(block: IncompleteBlock, s: *Sema, scratch_top: usize) !Air.Inst.Index {
570 s.setInstData(.block, block.block_index, try s.scratchToIndexList(scratch_top));
571 return block.block_index;
572 }
573};
574
575fn addBlock(s: *Sema) !IncompleteBlock {
576 log.info("start block at @{d}", .{s.air_instructions.len});
577 try s.air_instructions.append(s.gpa, .{
578 .tag = .block,
579 .data = undefined,
580 .flags = .{},
581 });
582 return .{ .block_index = @enumFromInt(s.air_instructions.len - 1) };
583}
584
585fn addInstRef(s: *Sema, comptime tag: Air.Inst.Tag, data: tag.Data()) !Air.Inst.Ref {
586 log.info("add air .{s}, {}", .{ @tagName(tag), data });
587 try s.air_instructions.append(s.gpa, .{
588 .tag = tag,
589 .data = Air.EncodedData.encode(tag, data),
590 .flags = .{},
591 });
592 const index: Air.Inst.Index = @enumFromInt(s.air_instructions.len - 1);
593 try s.scratch.append(s.arena, @intFromEnum(index));
594 return index.toRef();
595}
596
597fn setInstData(
598 s: *Sema,
599 comptime tag: Air.Inst.Tag,
600 index: Air.Inst.Index,
601 data: tag.Data(),
602) void {
603 const i = @intFromEnum(index);
604 assert(s.air_instructions.items(.tag)[i] == tag);
605 s.air_instructions.items(.data)[i] = Air.EncodedData.encode(tag, data);
606}
607
608fn makeValue(s: *Sema, key: InternPool.Key) !Value {
609 return .{ .ip_index = try s.ip.get(s.gpa, key) };
610}
611
612fn makeValueAir(s: *Sema, key: InternPool.Key) !Air.Inst.Ref {
613 return (try s.makeValue(key)).ip_index.toAir();
614}
615
616fn mapTeg(s: *Sema, inst: Teg.Inst.Ref) Air.Inst.Ref {
617 if (inst.toInterned()) |interned| {
618 return interned.toAir();
619 }
620 const index = inst.toIndex().?;
621 return s.inst_map.get(index) orelse {
622 if (std.debug.runtime_safety) {
623 std.debug.panic("Unmapped operand %{d}", .{@intFromEnum(index)});
624 }
625 unreachable;
626 };
627}
628
629fn validateType(s: *Sema, checked_type: Value, wanted_type: InternPool.Index) !void {
630 if (checked_type.toIntern() != wanted_type) {
631 log.info("failed type check {}, got {}", .{ Value.fromInterned(wanted_type).fmt(s.ip), checked_type.fmt(s.ip) });
632 return error.TypeCheckFailed;
633 } else {
634 log.info("passed type check {}", .{checked_type.fmt(s.ip)});
635 }
636}
637
638fn scratchToIndexList(s: *Sema, scratch_top: usize) !Air.IndexList {
639 const items = s.scratch.items[scratch_top..];
640 const extra_top = s.air_extra.list.items.len;
641 try s.air_extra.list.appendSlice(s.gpa, s.scratch.items[scratch_top..]);
642 return .{
643 .start = @intCast(extra_top),
644 .len = @intCast(items.len),
645 };
646}
647
648fn tempAir(s: *const Sema) Air {
649 return .{
650 .insts = s.air_instructions.slice(),
651 .extra = s.air_extra.constant(),
652 };
653}
654
655fn typeOfRef(s: *Sema, ref: Air.Inst.Ref) InternPool.Index {
656 return s.tempAir().typeOf(ref, s.ip);
657}
658
659fn lookupDecl(s: *Sema, name: NullTerminatedString) ?InternPool.DeclIndex {
660 var nsi = s.namespace_index;
661 while (true) {
662 const ns = s.ip.namespacePtr(nsi);
663 if (ns.lookupDirect(s.comp, name)) |decl_id| {
664 return decl_id;
665 }
666 nsi = ns.parent.unwrap() orelse
667 unreachable; // transform ensures all identifiers get resolved
668 }
669}
670
671const Sema = @This();
672
673const std = @import("std");
674const mem = std.mem;
675const Allocator = std.mem.Allocator;
676const assert = std.debug.assert;
677const log = std.log.scoped(.sema);
678
679const Comp = @import("Comp.zig");
680const File = Comp.File;
681const Decl = Comp.Decl;
682const Namespace = Comp.Namespace;
683
684const Air = @import("Air.zig");
685
686const Teg = @import("Teg.zig");
687
688const Value = @import("Value.zig");
689const InternPool = @import("InternPool.zig");
690const NullTerminatedString = InternPool.NullTerminatedString;
691const dod = @import("dod.zig");
692
693const tracer = @import("tracer");
src/compiler/Teg.Dump.zig created+292
......@@ -0,0 +1,292 @@
1//! This implements a debug formatter for Teg
2teg: Teg,
3
4const State = struct {
5 arena: std.mem.Allocator,
6 teg: *const Teg,
7 indent: usize = 0,
8 tty: std.io.tty.Config,
9 visited: std.AutoHashMapUnmanaged(Teg.Inst.Index, void) = .{},
10
11 fn newline(state: State, writer: anytype) !void {
12 try writer.writeByte('\n');
13 if (state.indent > 0)
14 try writer.writeByteNTimes(' ', state.indent * 2);
15 }
16
17 fn dumpInst(s: *State, writer: anytype, inst: Teg.Inst.Index) @TypeOf(writer).Error!void {
18 const teg = s.teg;
19 const tty = s.tty;
20
21 try writer.print("%{} = ", .{@intFromEnum(inst)});
22
23 const tag = s.teg.instTag(inst);
24
25 try tty.setColor(writer, .cyan);
26 try writer.print("{s}", .{@tagName(tag)});
27 try tty.setColor(writer, .reset);
28
29 if ((try s.visited.fetchPut(s.arena, inst, {})) != null) {
30 try tty.setColor(writer, .red);
31 try writer.writeAll(" [error: recursive tree]");
32 try tty.setColor(writer, .reset);
33 return;
34 }
35
36 switch (tag) {
37 .module => {
38 try s.dumpInstList(writer, teg.moduleDeclarations(inst), '{', '}');
39 },
40 .declaration => {
41 const decl_info = teg.declData(inst);
42 try writer.print(" ({s}) ", .{@tagName(decl_info.kind)});
43
44 const name = decl_info.name.slice(teg.string_bytes);
45 if (name.len == 0) {
46 try tty.setColor(writer, .bold);
47 try writer.writeAll("<top level code>");
48 } else {
49 try tty.setColor(writer, .magenta);
50 try writer.writeAll(name);
51 }
52 try tty.setColor(writer, .reset);
53
54 try writer.writeAll(" {");
55 s.indent += 1;
56 try s.newline(writer);
57
58 switch (decl_info.kind) {
59 .function => {
60 const fn_data: Teg.Inst.DeclFn = teg.fullFuncDecl(inst).@"1";
61 try writer.writeAll("params: ");
62 try s.dumpRefList(writer, fn_data.params.slice(teg), '[', ']');
63 try s.newline(writer);
64 try writer.writeAll("return_type: ");
65 try s.dumpRef(writer, fn_data.return_type);
66 try s.newline(writer);
67 try writer.writeAll("body: ");
68 try s.dumpRef(writer, fn_data.body);
69 },
70 }
71 s.indent -= 1;
72 try s.newline(writer);
73 try writer.writeAll("}");
74 },
75 .param => {
76 const data = teg.instData(.param, inst);
77 try writer.writeAll("(");
78 try tty.setColor(writer, .yellow);
79 try writer.print("{d}", .{data.data});
80 try tty.setColor(writer, .reset);
81 try writer.writeAll(")");
82 },
83 .block => {
84 const block = teg.instData(.block, inst);
85 const index_list = block.extra_index.get(teg);
86 try s.dumpInstList(writer, index_list.slice(teg), '[', ']');
87 },
88 .number => {
89 const number = teg.instData(.number, inst);
90 const float = number.extra_index.get(teg).decode();
91 try tty.setColor(writer, .yellow);
92 try writer.print(" {d}", .{float});
93 try tty.setColor(writer, .reset);
94 },
95 inline .field,
96 .store,
97 .add,
98 .sub,
99 .mul,
100 .div,
101 .rem,
102 .pow,
103 .@"and",
104 .@"or",
105 .bit_and,
106 .bit_or,
107 .bit_xor,
108 .shl,
109 .shr,
110 .eq,
111 .ne,
112 .lt,
113 .le,
114 .gt,
115 .ge,
116 => |comptime_tag| {
117 const data = teg.instData(comptime_tag, inst).extra_index.get(teg);
118 try writer.writeAll("(");
119 try s.dumpRefMarker(writer, data.lhs);
120 try writer.writeAll(", ");
121 try s.dumpRefMarker(writer, data.rhs);
122 try writer.writeAll(")");
123 },
124 .@"break" => {
125 const data: Teg.Inst.Break = teg.instData(.@"break", inst).extra_index.get(teg);
126 try writer.writeAll("(");
127 try s.dumpRefMarker(writer, data.block.toRef());
128 try writer.writeAll(", ");
129 try s.dumpRefMarker(writer, data.operand);
130 try writer.writeAll(")");
131 },
132 .string => {
133 const data: Teg.SlicedString = teg.instData(.string, inst).extra_index.get(teg);
134 try tty.setColor(writer, .green);
135 try writer.writeAll(" \"");
136 try writer.writeAll(data.slice(teg.string_bytes));
137 try writer.writeAll("\"");
138 try tty.setColor(writer, .reset);
139 },
140 inline .memory,
141 .discard,
142 .ensure_void,
143 .ensure_bool,
144 .negate,
145 .bool_not,
146 .bit_not,
147 .js,
148 .import,
149 .fetch,
150 .compile_error,
151 => |comptime_tag| {
152 const data = teg.instData(comptime_tag, inst);
153 try writer.writeAll("(");
154 try s.dumpRefMarker(writer, data.operand);
155 try writer.writeAll(")");
156 },
157 .call => {
158 const call: Teg.Inst.Call = teg.instData(.call, inst).extra_index.get(teg);
159 try writer.writeAll("(");
160 try s.dumpRefMarker(writer, call.base);
161 try writer.writeAll(", ");
162 try s.dumpRefMarkerList(writer, call.args.slice(teg), '[', ']');
163 try writer.writeAll(")");
164 },
165 .load_decl => {
166 const call: Teg.Inst.StrNode = teg.instData(.load_decl, inst);
167
168 try tty.setColor(writer, .red);
169 try writer.print(" {s}", .{call.string.slice(teg.string_bytes)});
170 try tty.setColor(writer, .reset);
171 },
172 .@"if" => {
173 const data: Teg.Inst.If = teg.instPayload(.@"if", inst);
174 try writer.writeAll("(");
175 try s.dumpRefMarker(writer, data.cond);
176 try writer.writeAll(") {");
177 s.indent += 1;
178 try s.newline(writer);
179 try writer.writeAll("yes: ");
180 try s.dumpRef(writer, data.yes);
181 try s.newline(writer);
182 try writer.writeAll("no: ");
183 try s.dumpRef(writer, data.no);
184 s.indent -= 1;
185 try s.newline(writer);
186 try writer.writeAll("}");
187 },
188 else => std.debug.panic("TODO format {s}", .{@tagName(tag)}),
189 }
190 }
191
192 fn dumpRef(s: *State, writer: anytype, ref: Teg.Inst.Ref) !void {
193 if (ref.toIndex()) |index| {
194 try s.dumpInst(writer, index);
195 } else {
196 try s.tty.setColor(writer, .blue);
197 try writer.print("%{s}", .{@tagName(ref)});
198 try s.tty.setColor(writer, .reset);
199 }
200 }
201
202 fn dumpRefMarker(s: *State, writer: anytype, ref: Teg.Inst.Ref) !void {
203 if (ref.toIndex()) |index| {
204 try s.tty.setColor(writer, .blue);
205 try writer.print("%{d}", .{@intFromEnum(index)});
206 try s.tty.setColor(writer, .reset);
207 } else {
208 try s.tty.setColor(writer, .blue);
209 try writer.print("%{s}", .{@tagName(ref)});
210 try s.tty.setColor(writer, .reset);
211 }
212 }
213
214 fn dumpInstList(state: *State, writer: anytype, list: []const Teg.Inst.Index, l: u8, r: u8) !void {
215 try writer.print(" {c}", .{l});
216 if (list.len > 0) {
217 state.indent += 1;
218
219 for (list) |inst| {
220 try state.newline(writer);
221 try state.dumpInst(writer, inst);
222 }
223
224 state.indent -= 1;
225 try state.newline(writer);
226 }
227 try writer.print("{c}", .{r});
228 }
229
230 fn dumpRefList(state: *State, writer: anytype, list: []const Teg.Inst.Ref, l: u8, r: u8) !void {
231 try writer.print(" {c}", .{l});
232 if (list.len > 0) {
233 state.indent += 1;
234
235 for (list) |ref| {
236 try state.newline(writer);
237 try state.dumpRef(writer, ref);
238 }
239
240 state.indent -= 1;
241
242 try state.newline(writer);
243 }
244 try writer.print("{c}", .{r});
245 }
246
247 fn dumpRefMarkerList(state: *State, writer: anytype, list: []const Teg.Inst.Ref, l: u8, r: u8) !void {
248 try writer.print(" {c}", .{l});
249 if (list.len > 0) {
250 state.indent += 1;
251
252 for (list) |ref| {
253 try state.newline(writer);
254 try state.dumpRefMarker(writer, ref);
255 }
256
257 state.indent -= 1;
258
259 try state.newline(writer);
260 }
261 try writer.print("{c}", .{r});
262 }
263};
264
265pub fn format(
266 dump: @This(),
267 comptime unused_fmt: []const u8,
268 _: std.fmt.FormatOptions,
269 writer: anytype,
270) !void {
271 comptime assert(unused_fmt.len == 0);
272
273 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
274 defer arena.deinit();
275
276 const teg = dump.teg;
277 var state = State{
278 .teg = &teg,
279 .tty = detectTTYFromWriter(writer),
280 .arena = arena.allocator(),
281 };
282 try state.dumpInst(writer, .main_module_inst);
283 try writer.writeAll("\n");
284 assert(state.indent == 0);
285}
286
287const std = @import("std");
288const assert = std.debug.assert;
289
290const Teg = @import("Teg.zig");
291
292const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter;
src/compiler/Teg.zig created+435
......@@ -0,0 +1,435 @@
1//! TODO: rename this to something else. Teg is not a graph
2
3insts: Inst.List.Slice,
4string_bytes: []const u8,
5extra: ExtraStore = .{},
6
7pub const ExtraStore = dod.ExtraStore(.teg, .@"const");
8pub const EncodedData = dod.EncodedData(Inst.Tag, u64);
9pub const ExtraIndex = ExtraStore.Index;
10pub const Dump = @import("Teg.Dump.zig");
11
12pub const Inst = struct {
13 pub const List = std.MultiArrayList(Inst);
14
15 tag: Tag,
16 data: EncodedData,
17
18 // todo: fully document every thing
19 pub const Tag = enum {
20 module,
21 declaration,
22
23 block,
24 /// Creates a memory cell in the current scope.
25 memory,
26 /// Performs a type coersion, such as in variable declarations.
27 /// Emits a sema error if the type conversion fails.
28 /// `let x: i32 = 1`
29 as,
30 load_decl,
31 /// Load a parameter
32 param,
33 store,
34 discard,
35 ensure_void,
36 ensure_bool,
37 /// lowered from `break` or `return`
38 /// ends a block with a resulting value
39 @"break",
40 number,
41 string,
42
43 @"if",
44
45 add,
46 sub,
47 mul,
48 div,
49 rem,
50 pow,
51 @"and",
52 @"or",
53 bit_and,
54 bit_or,
55 bit_xor,
56 shl,
57 shr,
58 eq,
59 ne,
60 lt,
61 le,
62 gt,
63 ge,
64
65 negate,
66 bool_not,
67 bit_not,
68 field,
69 call,
70
71 /// Implements the `@js` builtin
72 js,
73 /// Implements the `@import` builtin
74 import,
75 /// Implements the `@fetch` builtin
76 fetch,
77 /// Implements the `@compileError` builtin
78 compile_error,
79
80 pub fn Data(comptime tag: Tag) type {
81 return switch (tag) {
82 .module => PlNode(IndexList),
83 .declaration => PlNode(Decl),
84 .block => PlNode(IndexList),
85 .as => PlNode(As),
86 .@"break" => PlNode(Break),
87 .number => PlNode(Float),
88 .string => PlNode(SlicedString),
89 .call => PlNode(Call),
90 .@"if" => PlNode(If),
91 .load_decl => StrNode,
92 .param => DataNode,
93
94 // Single instruction operand
95 .memory,
96 .discard,
97 .ensure_void,
98 .ensure_bool,
99 .negate,
100 .bool_not,
101 .bit_not,
102 .js,
103 .import,
104 .fetch,
105 .compile_error,
106 => RefNode,
107
108 // Two instruction operands
109 .field,
110 .store,
111 .add,
112 .sub,
113 .mul,
114 .div,
115 .rem,
116 .pow,
117 .@"and",
118 .@"or",
119 .bit_and,
120 .bit_or,
121 .bit_xor,
122 .shl,
123 .shr,
124 .eq,
125 .ne,
126 .lt,
127 .le,
128 .gt,
129 .ge,
130 => PlNode(Bin),
131 };
132 }
133
134 pub fn airTag(comptime tag: Tag) Air.Inst.Tag {
135 return @field(Air.Inst.Tag, @tagName(tag));
136 }
137 };
138
139 /// TODO: make enum? idk
140 const NodeOffset = i32;
141
142 pub fn PlNode(comptime Payload: type) type {
143 return struct {
144 pub const T = Payload;
145
146 src_node: NodeOffset,
147 extra_index: ExtraIndex(Payload),
148 };
149 }
150
151 pub const RefNode = struct {
152 src_node: NodeOffset,
153 operand: Inst.Ref,
154 };
155
156 pub const StrNode = struct {
157 src_node: NodeOffset,
158 string: NullTerminatedString,
159 };
160
161 pub const DataNode = struct {
162 src_node: NodeOffset,
163 data: u32,
164 };
165
166 /// The position of a Teg instruction within the `Teg` instructions array.
167 pub const Index = enum(u32) {
168 /// Teg is structured so that the outermost "main" struct of any file
169 /// is always at index 0.
170 main_module_inst = 0,
171 _,
172
173 pub const static_len = @intFromEnum(Ref.empty_string) + 1;
174
175 pub fn toRef(i: Index) Inst.Ref {
176 return @enumFromInt(static_len + @intFromEnum(i));
177 }
178
179 pub fn toOptional(i: Index) OptionalIndex {
180 return @enumFromInt(@intFromEnum(i));
181 }
182 };
183
184 pub const OptionalIndex = enum(u32) {
185 /// Teg is structured so that the outermost "main" module of any file
186 /// is always at index 0.
187 main_module_inst = 0,
188 ref_start_index = Index.static_len,
189 none = std.math.maxInt(u32),
190 _,
191
192 pub fn unwrap(oi: OptionalIndex) ?Index {
193 return if (oi == .none) null else @enumFromInt(@intFromEnum(oi));
194 }
195 };
196
197 /// A reference to instruction, or to an InternPool index, or neither.
198 ///
199 /// If the integer tag value is < InternPool.static_len, then it
200 /// corresponds to an InternPool index. Otherwise, this refers to a ZIR
201 /// instruction.
202 ///
203 /// The tag type is specified so that it is safe to bitcast between `[]u32`
204 /// and `[]Ref`.
205 pub const Ref = enum(u32) {
206 opaque_type,
207 bool_type,
208 int32_type,
209 float_type,
210 string_type,
211 void_type,
212 noreturn_type,
213 type_type,
214
215 void_value,
216 bool_true,
217 bool_false,
218 int32_zero,
219 float_zero,
220 nan,
221 empty_string,
222
223 /// This Ref does not correspond to any instruction or constant
224 /// value and may instead be used as a sentinel to indicate null.
225 none = std.math.maxInt(u32),
226
227 _,
228
229 pub fn toIndex(inst: Ref) ?Index {
230 assert(inst != .none);
231 const ref_int = @intFromEnum(inst);
232 if (ref_int >= Index.static_len) {
233 return @enumFromInt(ref_int - Index.static_len);
234 } else {
235 return null;
236 }
237 }
238
239 pub fn toIndexAllowNone(inst: Ref) ?Index {
240 if (inst == .none) return null;
241 return toIndex(inst);
242 }
243
244 pub fn toInterned(inst: Ref) ?InternPool.Index {
245 const ref_int = @intFromEnum(inst);
246
247 // todo: is this > or >=
248 if (ref_int > Index.static_len)
249 return null;
250
251 return @enumFromInt(ref_int);
252 }
253 };
254
255 pub const Bin = struct {
256 lhs: Inst.Ref,
257 rhs: Inst.Ref,
258 };
259
260 pub const As = struct {
261 type: Inst.Ref,
262 operand: Inst.Ref,
263 };
264
265 pub const Break = struct {
266 block: Inst.Index,
267 operand: Inst.Ref,
268 };
269
270 pub const Call = struct {
271 base: Inst.Ref,
272 args: RefList,
273 };
274
275 pub const If = struct {
276 cond: Inst.Ref,
277 yes: Inst.Ref,
278 no: Inst.Ref,
279 };
280
281 /// This will contain trailing data depending on the .kind
282 pub const Decl = struct {
283 name: NullTerminatedString,
284 hash: Cache.BinDigest,
285 kind: Kind,
286 instruction_count: u32,
287
288 pub const Kind = enum {
289 function,
290 // future: struct, let, var, etc
291 };
292 };
293
294 /// Trailng data for function declarations
295 pub const DeclFn = struct {
296 params: RefList,
297 return_type: Ref,
298 body: Ref,
299 };
300};
301
302pub const NullTerminatedString = enum(u32) {
303 empty = 0,
304 _,
305
306 pub fn slice(str: NullTerminatedString, string_bytes: []const u8) [:0]const u8 {
307 if (str == .empty) return "";
308 const sub = string_bytes[@intFromEnum(str)..];
309 const i = mem.indexOfScalar(u8, sub, 0) orelse unreachable;
310
311 return sub[0..i :0];
312 }
313};
314
315pub const SlicedString = struct {
316 start: u32,
317 len: u32,
318
319 pub fn slice(str: SlicedString, string_bytes: []const u8) []const u8 {
320 return string_bytes[str.start..][0..str.len];
321 }
322};
323
324pub const RefList = struct {
325 start: u32,
326 len: u32,
327
328 pub fn slice(self: RefList, teg: *const Teg) []const Inst.Ref {
329 return @ptrCast(teg.extra.list.items[self.start..][0..self.len]);
330 }
331};
332
333pub const IndexList = struct {
334 start: u32,
335 len: u32,
336
337 pub fn slice(self: IndexList, teg: *const Teg) []const Inst.Index {
338 return @ptrCast(teg.extra.list.items[self.start..][0..self.len]);
339 }
340};
341
342pub fn deinit(teg: *Teg, gpa: std.mem.Allocator) void {
343 gpa.free(teg.string_bytes);
344 teg.extra.deinit(gpa);
345 teg.insts.deinit(gpa);
346 teg.* = undefined;
347}
348
349pub fn instTag(teg: *const Teg, index: Inst.Index) Inst.Tag {
350 return teg.insts.items(.tag)[@intFromEnum(index)];
351}
352
353pub fn instEncodedData(teg: *const Teg, index: Inst.Index) EncodedData {
354 return teg.insts.items(.data)[@intFromEnum(index)];
355}
356
357pub fn instData(teg: *const Teg, comptime tag: Inst.Tag, index: Inst.Index) tag.Data() {
358 return teg.instEncodedData(index).decode(tag, teg.instTag(index));
359}
360
361pub fn instPayload(teg: *const Teg, comptime tag: Inst.Tag, index: Inst.Index) tag.Data().T {
362 return teg.instData(tag, index).extra_index.get(teg);
363}
364
365pub fn blockInstructions(teg: *const Teg, index: Inst.Index) []const Inst.Index {
366 return teg.instData(.block, index).extra_index.get(teg).slice(teg);
367}
368
369pub fn fullFuncDecl(teg: *const Teg, index: Inst.Index) struct { Inst.Decl, Inst.DeclFn } {
370 const decl, var trail = teg.instData(.declaration, index).extra_index.getTrail(teg);
371 const decl_fn = trail.read(&teg.extra, Inst.DeclFn);
372 return .{ decl, decl_fn };
373}
374
375pub fn declData(teg: *const Teg, index: Inst.Index) Inst.Decl {
376 return teg.instData(.declaration, index).extra_index.get(teg);
377}
378
379pub fn moduleDeclarations(teg: *const Teg, module_index: Inst.Index) []const Inst.Index {
380 const module = teg.instData(.module, module_index);
381 const module_data: IndexList = module.extra_index.get(teg);
382 const slice = module_data.slice(teg);
383 // if (std.debug.runtime_safety) {
384 // for (slice) |item| {
385 // if (teg.instTag(item) != .declaration) {
386 // std.debug.print("\ngot unknown tag {s}\n", .{@tagName(teg.instTag(item))});
387 // unreachable;
388 // }
389 // }
390 // }
391 return slice;
392}
393
394pub fn rootFunction(teg: *const Teg) Inst.Index {
395 const decls = teg.moduleDeclarations(.main_module_inst);
396 const root_fn = decls[decls.len - 1];
397 assert(teg.declName(root_fn).len == 0);
398 return root_fn;
399}
400
401pub fn declName(teg: *const Teg, decl_index: Inst.Index) []const u8 {
402 const decl = teg.instData(.declaration, decl_index);
403 const decl_data: Inst.Decl = decl.extra_index.get(teg);
404 return decl_data.name.slice(teg.string_bytes);
405}
406
407pub fn dumpToWriter(teg: Teg, writer: anytype) !void {
408 try writer.print("{}", .{Dump{ .teg = teg }});
409}
410
411const Teg = @This();
412
413const std = @import("std");
414const mem = std.mem;
415const assert = std.debug.assert;
416
417const Cache = @import("Cache.zig");
418
419const Ast = @import("Ast.zig");
420const Unused = Ast.Unused;
421const NodeIndex = Ast.NodeIndex;
422const TokenIndex = Ast.TokenIndex;
423const OptionalNodeIndex = Ast.OptionalNodeIndex;
424const OptionalTokenIndex = Ast.OptionalTokenIndex;
425const OptionalExtraIndex = Ast.OptionalExtraIndex;
426const optionalTokenIndex = OptionalTokenIndex.init;
427const optionalNodeIndex = OptionalNodeIndex.init;
428const optionalExtraIndex = OptionalExtraIndex.init;
429
430const dod = @import("dod.zig");
431const Float = dod.Float;
432
433const InternPool = @import("InternPool.zig");
434
435const Air = @import("Air.zig");
src/compiler/Tokenizer.zig created+913
......@@ -0,0 +1,913 @@
1//! The tokenizer is extremely simple structure that, given a view of source
2//! code, can stream tokens one at a time. A token is an 8-bit tag and a 32-bit
3//! offset to where that token is located.
4//!
5//! Retriving payloads like string/numeric values, identifier names, etc is
6//! done separately, either in Parse or Transform, when the respective data
7//! is actually needed. Tokenization does guarante that such data is valid.
8
9// TODO: make `code` a [:0]const u8. this doesnt disallow null bytes, but
10// it requires the source to end with one. This can make byteAt(1) return u8
11// instead of an optional. I am told this improves performance.
12code: []const u8,
13pos: ByteOffset = 0,
14
15/// Tokenize source code into a list of tokens.
16/// When an error occurs, the position of the error is returned in `error_pos`.
17/// Caller owns returned slice.
18pub fn tokenize(
19 source_code: []const u8,
20 gpa: mem.Allocator,
21 error_pos: *ByteOffset,
22) TokenizeError!Token.List.Slice {
23 const trace = T.trace(@src());
24 defer trace.end();
25
26 var tokens = Token.List{};
27 errdefer tokens.deinit(gpa);
28
29 // pre-allocate some space based on a very rough guess
30 try tokens.ensureTotalCapacity(gpa, @divFloor(source_code.len, 8));
31
32 var tokenizer: Tokenizer = .{ .code = source_code, .pos = 0 };
33 errdefer error_pos.* = tokenizer.pos;
34
35 tokenizer.skipWhitespace();
36 while (try tokenizer.next()) |token| {
37 try tokens.append(gpa, token);
38 tokenizer.skipWhitespace();
39 }
40
41 try tokens.append(gpa, .{
42 .tag = .eof,
43 .offset = @intCast(source_code.len),
44 });
45
46 return tokens.toOwnedSlice();
47}
48
49pub const ByteOffset = u32;
50
51pub const Token = struct {
52 tag: Tag,
53 offset: ByteOffset,
54
55 pub const List = std.MultiArrayList(Token);
56
57 pub const Tag = enum {
58 id,
59 builtin_id,
60 number_int,
61 number_float,
62 string,
63 multiline_string,
64 doc_comment,
65 module_doc_comment,
66
67 // sort-lines: start
68 kw_assert,
69 kw_break,
70 kw_comptime,
71 kw_defer,
72 kw_else,
73 kw_enum,
74 kw_fn,
75 kw_infer,
76 kw_for,
77 kw_while,
78 kw_if,
79 kw_impl,
80 kw_json,
81 kw_let,
82 kw_match,
83 kw_module,
84 kw_pub,
85 kw_pure,
86 kw_return,
87 kw_struct,
88 kw_trait,
89 kw_var,
90 // sort-lines: end
91
92 // sort-lines: start
93 @"and",
94 and_and,
95 and_and_equal,
96 and_equal,
97 bang,
98 bang_equal,
99 caret,
100 caret_equal,
101 colon,
102 comma,
103 dot,
104 end_brace,
105 end_bracket,
106 end_paren,
107 equal,
108 equal_equal,
109 equal_lt,
110 gt,
111 gt_equal,
112 gt_gt,
113 gt_gt_equal,
114 gt_gt_gt,
115 gt_gt_gt_equal,
116 lt,
117 lt_equal,
118 lt_lt,
119 lt_lt_equal,
120 minus,
121 minus_equal,
122 minus_lt,
123 percent,
124 percent_equal,
125 pipe,
126 pipe_equal,
127 pipe_pipe,
128 pipe_pipe_equal,
129 plus,
130 plus_equal,
131 semi,
132 slash,
133 slash_equal,
134 star,
135 star_equal,
136 star_star,
137 star_star_equal,
138 start_brace,
139 start_bracket,
140 start_paren,
141 tilde,
142 tilde_equal,
143 // sort-lines: end
144
145 /// Represents a tokenization error. Instead of reporting an error during
146 /// tokenization, parsing will begin, and an error will be reported when
147 /// this token is hit. That way, the error can tell you what it expected.
148 invalid,
149 /// End of file
150 /// Used so that parser does not need to do out of bounds checks
151 eof,
152
153 /// Most tokens have a static length, such as "struct" or "+="
154 pub fn staticLen(tag: Tag) ?u32 {
155 return switch (tag) {
156 // do not have a static length
157 .id,
158 .builtin_id,
159 .number_int,
160 .number_float,
161 .string,
162 .multiline_string,
163 .doc_comment,
164 .module_doc_comment,
165 => null,
166
167 .@"and" => 1,
168 .and_and => 2,
169 .bang => 1,
170 .bang_equal => 2,
171 .equal_lt => 2,
172 .caret => 1,
173 .end_brace => 1,
174 .end_bracket => 1,
175 .end_paren => 1,
176 .colon => 1,
177 .comma => 1,
178 .dot => 1,
179 .equal => 1,
180 .equal_equal => 2,
181 .gt => 1,
182 .gt_equal => 2,
183 .gt_gt => 2,
184 .gt_gt_gt => 3,
185 .lt => 1,
186 .lt_equal => 2,
187 .lt_lt => 2,
188 .minus => 1,
189 .start_brace => 1,
190 .start_bracket => 1,
191 .start_paren => 1,
192 .pipe => 1,
193 .pipe_pipe => 2,
194 .percent => 1,
195 .plus => 1,
196 .minus_lt => 2,
197 .semi => 1,
198 .slash => 1,
199 .star => 1,
200 .star_star => 2,
201 .tilde => 1,
202 .slash_equal => 2,
203 .percent_equal => 2,
204 .tilde_equal => 2,
205 .caret_equal => 2,
206 .and_equal => 2,
207 .pipe_equal => 2,
208 .lt_lt_equal => 3,
209 .gt_gt_equal => 3,
210 .gt_gt_gt_equal => 4,
211 .plus_equal => 2,
212 .minus_equal => 2,
213 .star_equal => 2,
214 .star_star_equal => 3,
215 .pipe_pipe_equal => 3,
216 .and_and_equal => 3,
217
218 .invalid => 0,
219 .eof => 0,
220
221 inline else => |keyword| comptime len: {
222 if (!mem.startsWith(u8, @tagName(keyword), "kw_")) {
223 @compileError("Token tag is not a keyword: " ++ @tagName(keyword));
224 }
225 break :len @tagName(keyword).len - 3;
226 },
227 };
228 }
229 };
230
231 /// Instead of storing the length of a token, we can recompute this
232 /// information on demand, as it is not often needed.
233 ///
234 /// If you are computing the length of something that is knowable
235 /// without the source code, use token.tag.staticLen() instead.
236 pub fn len(token: Token, utf8_source: []const u8) u32 {
237 return token.tag.staticLen() orelse {
238 var tokenizer = Tokenizer{ .code = utf8_source, .pos = token.offset };
239 assert(
240 (tokenizer.next() catch
241 unreachable // data corruption
242 orelse unreachable // data corruption
243 ).tag == token.tag,
244 );
245 return tokenizer.pos - token.offset;
246 };
247 }
248
249 pub fn slice(token: Token, utf8_source: []const u8) []const u8 {
250 return utf8_source[token.offset..][0..token.len(utf8_source)];
251 }
252
253 const keywords = std.ComptimeStringMap(Tag, keywords: {
254 const fields = std.enums.values(Tag);
255 const KV = struct { []const u8, Tag };
256 var kws: [fields.len]KV = undefined;
257 var i = 0;
258 for (fields) |field| {
259 if (mem.startsWith(u8, @tagName(field), "kw_")) {
260 kws[i] = .{ @tagName(field)[3..], field };
261 i += 1;
262 }
263 }
264 break :keywords kws[0..i];
265 });
266};
267
268/// When any of these error occurs, the tokenizer's position is set to
269/// the start of the offending token.
270pub const NextTokenError = error{
271 /// Generic token syntax error
272 InvalidSyntax,
273 /// A trailing '_' was seen in a numeric literal.
274 TrailingNumericSeparator,
275 /// Newline or EOF encountered while parsing a string literal.
276 UnterminatedString,
277 /// A string escape sequence was invalid.
278 InvalidEscapeSequence,
279 /// The codepoint represented by a unicode escape sequence
280 /// is not a valid unicode codepoint.
281 InvalidEscapeCodepoint,
282 /// There are unresolved git conflicts in the source code.
283 GitConflict,
284 /// The source code is not valid UTF-8.
285 InvalidUtf8,
286};
287
288pub const TokenizeError = mem.Allocator.Error || NextTokenError;
289
290/// Next assumes the position is already at the start of a token.
291/// You must call `skipWhitespace` before calling this.
292pub fn next(t: *Tokenizer) !?Token {
293 while (true) {
294 const trace = T.trace(@src());
295 defer trace.end();
296
297 switch (t.byte() orelse return null) {
298 ' ', '\t', '\n', '\r' => unreachable, // call skipWhitespace
299
300 'a'...'z', 'A'...'Z', '_', '$' => {
301 const start = t.pos;
302 t.pos += 1;
303 return try t.tokenizeIdentifier(start);
304 },
305
306 '@' => switch (t.byteAt(1) orelse 0) {
307 '"' => {
308 const start = t.pos;
309 t.pos += 1;
310 try t.consumeString();
311 return .{ .tag = .id, .offset = start };
312 },
313 else => {
314 const start = t.pos;
315 t.pos += 1;
316 return try t.tokenizeIdentifier(start);
317 },
318 },
319
320 '0'...'9' => return try t.tokenizeNumber(),
321
322 '.' => switch (t.byteAt(1) orelse 0) {
323 '0'...'9' => return try t.tokenizeNumber(),
324 else => return t.simpleToken(.dot),
325 },
326
327 '/' => switch (t.byteAt(1) orelse 0) {
328 '=' => return t.simpleToken(.slash_equal),
329 '/' => {
330 if ((t.byteAt(2) orelse 0 == '/') and
331 (t.byteAt(3) orelse 0 != '/'))
332 {
333 const start = t.pos;
334 t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 3, '\n') orelse t.code.len);
335 return .{ .tag = .doc_comment, .offset = start };
336 }
337
338 if (t.byteAt(2) orelse 0 == '!') {
339 const start = t.pos;
340 t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len);
341 return .{ .tag = .module_doc_comment, .offset = start };
342 }
343
344 // skip the comment
345 t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len);
346 t.skipWhitespace();
347 continue;
348 },
349 else => return t.simpleToken(.slash),
350 },
351
352 '\\' => switch (t.byteAt(1) orelse 0) {
353 '\\' => {
354 const start = t.pos;
355 t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len);
356 return .{ .tag = .multiline_string, .offset = start };
357 },
358 else => return t.maybeUTF8Error(error.InvalidSyntax),
359 },
360
361 '"' => return try t.tokenizeString(),
362
363 '&' => switch (t.byteAt(1) orelse 0) {
364 '&' => return t.simpleToken(.and_and),
365 '=' => return t.simpleToken(.and_equal),
366 else => return t.simpleToken(.@"and"),
367 },
368 '|' => switch (t.byteAt(1) orelse 0) {
369 '|' => return t.simpleToken(.pipe_pipe),
370 '=' => return t.simpleToken(.pipe_equal),
371 else => return t.simpleToken(.pipe),
372 },
373 '^' => switch (t.byteAt(1) orelse 0) {
374 '^' => return t.simpleToken(.caret),
375 '=' => return t.simpleToken(.caret_equal),
376 else => return t.simpleToken(.caret),
377 },
378 '!' => switch (t.byteAt(1) orelse 0) {
379 '=' => return t.simpleToken(.equal_equal),
380 else => return t.simpleToken(.bang_equal),
381 },
382 '=' => switch (t.byteAt(1) orelse 0) {
383 '=' => return t.simpleToken(.equal_equal),
384 else => return t.simpleToken(.equal),
385 },
386 '>' => switch (t.byteAt(1) orelse 0) {
387 '=' => return t.simpleToken(.gt_equal),
388 '>' => switch (t.byteAt(2) orelse 0) {
389 '>' => return t.simpleToken(.gt_gt_gt),
390 else => return t.simpleToken(.gt_gt),
391 },
392 else => return t.simpleToken(.gt),
393 },
394 '<' => switch (t.byteAt(1) orelse 0) {
395 '=' => return t.simpleToken(.lt_equal),
396 '<' => switch (t.byteAt(2) orelse 0) {
397 '=' => return t.simpleToken(.lt_lt_equal),
398 else => {
399 // Git merge conflicts are <<<<<<<
400 const start = t.pos;
401 t.pos += 2;
402 brk: {
403 var remain: u3 = 4;
404 while (t.byte() orelse break :brk == '<') {
405 t.pos += 1;
406 remain -= 1;
407 if (remain == 0) {
408 return error.GitConflict;
409 }
410 }
411 }
412 t.pos = start;
413 return t.simpleToken(.lt_lt);
414 },
415 },
416 else => return t.simpleToken(.lt),
417 },
418 '+' => switch (t.byteAt(1) orelse 0) {
419 '=' => return t.simpleToken(.plus_equal),
420 else => return t.simpleToken(.plus),
421 },
422 '-' => switch (t.byteAt(1) orelse 0) {
423 '=' => return t.simpleToken(.minus_equal),
424 '>' => return t.simpleToken(.minus_lt),
425 else => return t.simpleToken(.minus),
426 },
427 '*' => switch (t.byteAt(1) orelse 0) {
428 '*' => return t.simpleToken(.star_star),
429 '=' => return t.simpleToken(.star_equal),
430 else => return t.simpleToken(.star),
431 },
432 '%' => switch (t.byteAt(1) orelse 0) {
433 '=' => return t.simpleToken(.percent_equal),
434 else => return t.simpleToken(.percent),
435 },
436 '~' => switch (t.byteAt(1) orelse 0) {
437 '=' => return t.simpleToken(.tilde_equal),
438 else => return t.simpleToken(.tilde),
439 },
440
441 ';' => return t.simpleToken(.semi),
442 ':' => return t.simpleToken(.colon),
443 '(' => return t.simpleToken(.start_paren),
444 ')' => return t.simpleToken(.end_paren),
445 '[' => return t.simpleToken(.start_bracket),
446 ']' => return t.simpleToken(.end_bracket),
447 '{' => return t.simpleToken(.start_brace),
448 '}' => return t.simpleToken(.end_brace),
449 ',' => return t.simpleToken(.comma),
450
451 0x80...0xFF => {
452 // need to decode utf-8
453 const byte_len, const code_point = try t.codePoint();
454
455 if (unicode_props.isXidStart(code_point)) {
456 const start = t.pos;
457 t.pos += byte_len;
458 return try t.tokenizeIdentifier(start);
459 }
460
461 return error.InvalidSyntax;
462 },
463
464 else => return error.InvalidSyntax,
465 }
466 }
467}
468
469fn byte(t: Tokenizer) ?u8 {
470 if (t.pos >= t.code.len) return null;
471 return t.code[t.pos];
472}
473
474/// Remember that using this only makes sense if you know for certain that all
475/// previous bytes are ASCII, otherwise you may hit UB
476fn byteAt(t: Tokenizer, offset: u32) ?u8 {
477 if (t.pos + offset >= t.code.len) return null;
478 // assert that the previous bytes are ASCII
479 // current byte is allowed to be a continuation byte
480 for (t.code[t.pos .. t.pos + offset]) |b| {
481 assert(b < 0x80);
482 }
483 return t.code[t.pos + offset];
484}
485
486fn codePoint(t: Tokenizer) !struct { u3, u21 } {
487 const b = t.byte() orelse unreachable; // must check the first byte manually
488 assert(b >= 0x80); // must be a continuation byte
489
490 const byte_len = unicode.utf8ByteSequenceLength(b) catch
491 return error.InvalidUtf8;
492 assert(byte_len > 1);
493
494 const code_point = unicode.utf8Decode(t.code[t.pos..][0..byte_len]) catch
495 return error.InvalidUtf8;
496
497 return .{ byte_len, code_point };
498}
499
500fn isAtIdentifierStart(t: *Tokenizer) !bool {
501 switch (t.byte() orelse return false) {
502 'a'...'z', 'A'...'Z', '_', '$' => return true,
503 else => return false,
504 0x80...0xFF => {
505 _, const code_point = try t.codePoint();
506 return unicode_props.isXidStart(code_point);
507 },
508 }
509}
510
511fn tokenizeIdentifier(t: *Tokenizer, start_byte: u32) !Token {
512 const trace = T.trace(@src());
513 defer trace.end();
514
515 while (true) {
516 switch (t.byte() orelse break) {
517 'a'...'z', 'A'...'Z', '0'...'9', '_', '$' => {
518 t.pos += 1;
519 },
520 0x80...0xFF => {
521 const byte_len, const code_point = try t.codePoint();
522
523 t.pos += byte_len;
524
525 if (!unicode_props.isXidContinue(code_point)) {
526 break;
527 }
528 },
529 else => break,
530 }
531 }
532
533 return .{
534 .tag = Token.keywords.get(t.code[start_byte..t.pos]) orelse
535 if (t.code[start_byte] == '@')
536 .builtin_id
537 else
538 .id,
539 .offset = start_byte,
540 };
541}
542
543/// Numbers have the following rules:
544/// - May start with a `.`, but cannot end with `.`
545/// - May contain leading zeros, but transform pass will warn.
546/// - May contain a `_` separator, cannot have two in a row.
547/// - May contain `e` or `E` for exponents.
548/// - May use 0x, 0b, 0o prefixes for hex, binary, octal numbers.
549/// The token is tagged as `number_int` or `number_float` depending
550/// on if it contained a `.` This is to help later re-parses.
551//
552// TODO: is it a zig compiler bug that `?` is required here?
553fn tokenizeNumber(t: *Tokenizer) !?Token {
554 const trace = T.trace(@src());
555 defer trace.end();
556
557 // Caller ensures the following invariant:
558 switch (t.byte() orelse unreachable) {
559 '.' => switch (t.byteAt(1) orelse unreachable) {
560 '0'...'9' => {},
561 else => unreachable,
562 },
563 '0'...'9' => {},
564 else => unreachable,
565 }
566
567 // Parse Hex, Binary, Octal numbers
568 if (t.byte() == '0') {
569 switch (t.byteAt(1) orelse 0) {
570 'x', 'X' => {
571 t.pos += 2;
572 try t.consumeIntegerRadix(.hex);
573 },
574 'b', 'B' => {
575 t.pos += 2;
576 try t.consumeIntegerRadix(.binary);
577 },
578 'o', 'O' => {
579 t.pos += 2;
580 try t.consumeIntegerRadix(.octal);
581 },
582 else => {},
583 }
584 }
585
586 const start = t.pos;
587 var tag: Token.Tag = .number_int;
588
589 if (t.byte() == '.') {
590 t.pos += 1;
591 tag = .number_float;
592 try t.consumeInteger();
593 } else {
594 try t.consumeInteger();
595
596 // decimal point notation
597 switch (t.byte() orelse 0) {
598 '.' => switch (t.byteAt(1) orelse 0) {
599 '0'...'9' => {
600 t.pos += 1;
601 tag = .number_float;
602 try t.consumeInteger();
603 },
604 else => {
605 // we have not advanced the tokenizer past the `.`
606 assert(tag == .number_int);
607 return .{ .tag = tag, .offset = start };
608 },
609 },
610 else => {},
611 }
612 }
613
614 switch (t.byte() orelse 0) {
615 'e', 'E' => {
616 tag = .number_float;
617 t.pos += 1;
618
619 switch (t.byte() orelse 0) {
620 '+', '-' => {
621 t.pos += 1;
622 switch (t.byte() orelse 0) {
623 '0'...'9' => {},
624 else => return t.maybeUTF8Error(error.InvalidSyntax),
625 }
626 },
627 '0'...'9' => {},
628 else => return t.maybeUTF8Error(error.InvalidSyntax),
629 }
630
631 try t.consumeInteger();
632
633 // decimal point notation
634 switch (t.byte() orelse 0) {
635 '.' => switch (t.byteAt(1) orelse 0) {
636 '0'...'9' => {
637 t.pos += 1;
638 tag = .number_float;
639 try t.consumeInteger();
640 },
641 else => {},
642 },
643 else => {},
644 }
645 },
646 else => {},
647 }
648
649 // after numbers must be a valid word break the above returns are right
650 // before parsing `.` so those can be unchecked.
651 if (try t.isAtIdentifierStart()) {
652 return error.InvalidSyntax;
653 }
654
655 return .{
656 .tag = tag,
657 .offset = start,
658 };
659}
660
661fn maybeConsumeInteger(t: *Tokenizer) !?void {
662 const trace = T.trace(@src());
663 defer trace.end();
664
665 switch (t.byte() orelse return null) {
666 '0'...'9' => {
667 t.pos += 1;
668 },
669 else => return null,
670 }
671 t.consumeInteger();
672}
673
674/// Requires the first byte to be a digit
675fn consumeInteger(t: *Tokenizer) !void {
676 const trace = T.trace(@src());
677 defer trace.end();
678
679 switch (t.byte() orelse unreachable) {
680 '0'...'9' => {
681 t.pos += 1;
682 },
683 else => unreachable,
684 }
685
686 while (true) {
687 switch (t.byte() orelse return) {
688 '0'...'9' => {
689 t.pos += 1;
690 },
691 '_' => {
692 t.pos += 1;
693 switch (t.byte() orelse 0) {
694 '0'...'9' => {
695 t.pos += 1;
696 },
697 else => return t.maybeUTF8Error(error.TrailingNumericSeparator),
698 }
699 },
700 else => break,
701 }
702 }
703}
704
705const Radix = enum { hex, binary, octal };
706
707fn consumeIntegerRadix(t: *Tokenizer, comptime radix: Radix) !void {
708 const trace = T.trace(@src());
709 defer trace.end();
710
711 const digits = switch (radix) {
712 .hex => "0123456789abcdefABCDEF",
713 .binary => "01",
714 .octal => "01234567",
715 };
716
717 first: {
718 const b = t.byte() orelse return;
719 inline for (digits) |digit| {
720 if (b == digit) {
721 break :first;
722 }
723 }
724 return t.maybeUTF8Error(error.InvalidSyntax);
725 }
726
727 t.pos += 1;
728
729 while (true) {
730 brk: {
731 const b = t.byte() orelse return;
732 inline for (digits) |digit| {
733 if (b == digit) {
734 break :brk;
735 }
736 }
737 return;
738 }
739 t.pos += 1;
740 }
741}
742
743fn tokenizeString(t: *Tokenizer) !Token {
744 assert(t.byte().? == '"');
745 const start = t.pos;
746 try t.consumeString();
747 return .{ .tag = .string, .offset = start };
748}
749
750fn consumeString(t: *Tokenizer) !void {
751 const trace = T.trace(@src());
752 defer trace.end();
753
754 assert(t.byte().? == '"');
755
756 t.pos += 1;
757
758 while (true) {
759 const b = t.byte() orelse return error.UnterminatedString;
760 switch (b) {
761 '"' => {
762 t.pos += 1;
763 break;
764 },
765 '\\' => {
766 switch (t.byteAt(1) orelse {
767 // we want error.InvalidEscapeSequence to be at the '\' char,
768 // but we want error.UnterminatedString to be at the end of the source code
769 t.pos += 1;
770 return error.UnterminatedString;
771 }) {
772 '\n' => return error.UnterminatedString,
773 '\\' => t.pos += 2,
774 '"' => t.pos += 2,
775 'n' => t.pos += 2,
776 'r' => t.pos += 2,
777 't' => t.pos += 2,
778 '0' => switch (t.byteAt(2) orelse {
779 t.pos += 2;
780 return error.UnterminatedString;
781 }) {
782 // '\0xFF' hex escape
783 'x' => {},
784 // '\0' is special cased
785 else => t.pos += 2,
786 },
787 'u' => {
788 const esc_start = t.pos;
789 t.consumeStringUnicodeEscape() catch |err| switch (err) {
790 error.InvalidEscapeSequence => {
791 if (t.byte().? == '\n')
792 return error.UnterminatedString;
793 t.pos = esc_start;
794 return err;
795 },
796 error.InvalidEscapeCodepoint => {
797 t.pos = esc_start;
798 return err;
799 },
800 error.UnterminatedString => return err,
801 };
802 },
803 else => return t.maybeUTF8ErrorAt(error.InvalidEscapeSequence, 1),
804 }
805 },
806 else => {
807 t.pos += 1;
808 },
809 0x80...0xFF => {
810 const byte_len, _ = try t.codePoint();
811 t.pos += byte_len;
812 },
813 }
814 }
815}
816
817fn consumeStringUnicodeEscape(t: *Tokenizer) !void {
818 const trace = T.trace(@src());
819 defer trace.end();
820
821 assert(t.byte().? == '\\');
822 assert(t.byteAt(1).? == 'u');
823 t.pos += 2;
824
825 if ((t.byte() orelse return error.UnterminatedString) != '{')
826 return error.InvalidEscapeSequence;
827 t.pos += 1;
828
829 if ((t.byte() orelse return error.UnterminatedString) == '0') {
830 if ((t.byteAt(1) orelse return error.UnterminatedString) == 'x') {
831 t.pos += 2;
832 }
833 }
834
835 const start = t.pos;
836 t.consumeIntegerRadix(.hex) catch return error.InvalidEscapeSequence;
837
838 const slice = t.code[start..t.pos];
839
840 if ((t.byte() orelse return error.UnterminatedString) != '}')
841 return error.InvalidEscapeSequence;
842
843 const code_point = std.fmt.parseInt(u21, slice, 16) catch |err| switch (err) {
844 error.Overflow => return error.InvalidEscapeCodepoint,
845 error.InvalidCharacter => unreachable,
846 };
847 _ = std.unicode.utf8CodepointSequenceLength(code_point) catch |err| switch (err) {
848 error.CodepointTooLarge => return error.InvalidEscapeCodepoint,
849 };
850
851 t.pos += 1;
852}
853
854fn simpleToken(t: *Tokenizer, comptime tag: Token.Tag) Token {
855 const start = t.pos;
856 t.pos += comptime (tag.staticLen() orelse unreachable);
857 return .{ .tag = tag, .offset = start };
858}
859
860/// We are about to return an error, such as error.TrailingNumericSeparator.
861/// However, the character we ran into might actually be invalid UTF-8.
862/// This utility just checks for that to make sure we dont mask error.InvalidUtf8
863fn maybeUTF8Error(t: *Tokenizer, comptime real_err: anytype) @Type(.{
864 .ErrorSet = &[_]std.builtin.Type.Error{
865 .{ .name = "InvalidUtf8" },
866 .{ .name = @errorName(real_err) },
867 },
868}) {
869 switch (t.byte() orelse return real_err) {
870 0x80...0xFF => {
871 _ = try t.codePoint();
872 return real_err;
873 },
874 else => return real_err,
875 }
876}
877
878fn maybeUTF8ErrorAt(t: *Tokenizer, comptime real_err: anytype, at: u32) @Type(.{
879 .ErrorSet = &[_]std.builtin.Type.Error{
880 .{ .name = "InvalidUtf8" },
881 .{ .name = @errorName(real_err) },
882 },
883}) {
884 switch (t.byteAt(at) orelse return real_err) {
885 0x80...0xFF => {
886 _ = try t.codePoint();
887 return real_err;
888 },
889 else => return real_err,
890 }
891}
892
893pub fn skipWhitespace(t: *Tokenizer) void {
894 const trace = T.trace(@src());
895 defer trace.end();
896
897 while (true) {
898 switch (t.byte() orelse break) {
899 ' ', '\t', '\n', '\r' => {
900 t.pos += 1;
901 },
902 else => break,
903 }
904 }
905}
906
907const Tokenizer = @This();
908const std = @import("std");
909const unicode = std.unicode;
910const assert = std.debug.assert;
911const mem = std.mem;
912const unicode_props = @import("unicode_props.zig");
913const T = @import("tracer");
src/compiler/Transform.zig created+1276
......@@ -0,0 +1,1276 @@
1//! Transform pass
2gpa: Allocator,
3ast: Ast,
4insts: Inst.List = .{},
5string_bytes: std.ArrayListUnmanaged(u8) = .{},
6extra: ExtraStore = .{},
7
8scratch: std.ArrayListUnmanaged(u32) = .{},
9
10// TODO: refactor this away
11// The new home for this memory should be pre-allocated on scratch,
12// but then remove the elements, moving scratch items on top of this
13// backwards a few.
14call_args: std.ArrayListUnmanaged(Inst.Ref) = .{},
15scopes: std.ArrayListUnmanaged(Scope) = .{},
16
17pub fn transform(ast: Ast, gpa: Allocator) !Teg {
18 const trace = tracer.trace(@src());
19 defer trace.end();
20
21 var t: Transform = .{
22 .ast = ast,
23 .gpa = gpa,
24 };
25 errdefer t.deinit();
26
27 try t.string_bytes.append(t.gpa, 0);
28
29 const module = try t.transformModule(0);
30 assert(@intFromEnum(module) == 0);
31
32 try t.insts.setCapacity(t.gpa, t.insts.len);
33
34 defer t.deinit();
35
36 return Teg{
37 .insts = t.insts.toOwnedSlice(),
38 .string_bytes = try t.string_bytes.toOwnedSlice(t.gpa),
39 .extra = try t.extra.toOwnedStore(t.gpa),
40 };
41}
42
43pub fn deinit(t: *Transform) void {
44 t.scratch.deinit(t.gpa);
45 t.call_args.deinit(t.gpa);
46 assert(t.scopes.items.len == 0);
47 t.scopes.deinit(t.gpa);
48 t.* = undefined;
49}
50
51const Scope = struct {
52 symbols: std.StringArrayHashMapUnmanaged(Symbol) = .{},
53
54 const Symbol = struct {
55 node: NodeIndex,
56 data: Symbol.Data,
57 used: bool = false,
58 mutated: bool = false,
59
60 const Data = union(enum) {
61 declaration,
62 param,
63 mutable: Inst.Ref,
64 immutable: Inst.Ref,
65 };
66 };
67
68 fn deinit(s: *Scope, gpa: Allocator) void {
69 s.symbols.deinit(gpa);
70 s.* = undefined;
71 }
72};
73
74const ScratchType = enum {
75 ref,
76 index,
77};
78
79const BlockContext = struct {
80 node_base: i32,
81 return_target: ?*?Inst.Index = null,
82 break_target: ?*BreakTarget = null,
83
84 /// This struct is always stored on the stack; the idea is to pass
85 /// mutable pointers to it. When `break :brk` is encountered
86 const BreakTarget = struct {
87 label: ?[]const u8,
88 /// This allows `continue` as well as implicit break targetting.
89 is_loop: bool,
90 /// If `null`, this label was unused.
91 block_index: ?Inst.Index = null,
92 parent: ?*BreakTarget = null,
93
94 fn addChild(parent: *BreakTarget, label: ?[]const u8, is_loop: bool) BreakTarget {
95 return .{
96 .parent = parent,
97 .label = label,
98 .is_loop = is_loop,
99 };
100 }
101
102 fn getInst(target: *BreakTarget, t: *Transform) !Inst.Index {
103 return target.block_index orelse {
104 const inst = try t.addRootInst(.{
105 .tag = .block,
106 .data = undefined,
107 });
108 target.block_index = inst;
109 return inst;
110 };
111 }
112 };
113
114 fn init(root_node: NodeIndex) BlockContext {
115 return .{
116 .node_base = @intCast(root_node),
117 };
118 }
119
120 fn nodeOffset(self: BlockContext, node: NodeIndex) i32 {
121 return @as(i32, @intCast(node)) - self.node_base;
122 }
123
124 fn declOffset(self: BlockContext, node: NodeIndex) BlockContext {
125 return .{
126 .node_base = self.nodeOffset(node),
127 };
128 }
129
130 fn withReturn(self: BlockContext, return_target: ?*?Inst.Index) BlockContext {
131 return .{
132 .node_base = self.node_base,
133 .return_target = return_target,
134 };
135 }
136
137 fn withBreak(self: BlockContext, break_target: *BreakTarget) BlockContext {
138 return .{
139 .node_base = self.node_base,
140 .return_target = self.return_target,
141 .break_target = break_target,
142 };
143 }
144
145 fn findAndMarkBreak(self: BlockContext, t: *Transform, label: []const u8) !?Inst.Index {
146 var break_target = self.break_target;
147 while (break_target) |target| : (break_target = target.parent) {
148 if (target.label) |target_label| {
149 if (mem.eql(u8, label, target_label)) {
150 return try target.getInst(t);
151 }
152 }
153 }
154 return null;
155 }
156};
157
158const Liveness = enum { alive, dead };
159
160const BuiltInIds = std.ComptimeStringMap(Inst.Ref, .{
161 .{ "void", .void_value },
162 .{ "true", .bool_true },
163 .{ "false", .bool_false },
164 .{ "int", .int32_type },
165 .{ "float", .float_type },
166 .{ "string", .string_type },
167 // .{ "bigint", .bigint_type },
168 .{ "bool", .bool_type },
169 .{ "noreturn", .noreturn_type },
170 .{ "type", .type_type },
171 // todo: nan/infinity?
172});
173
174fn transformModule(t: *Transform, module_node: NodeIndex) !Inst.Index {
175 const trace = tracer.trace(@src());
176 defer trace.end();
177
178 const module = try t.addRootInst(.{
179 .tag = .module,
180 .data = undefined,
181 });
182
183 const decl = BlockContext.init(module_node);
184
185 try t.pushScope();
186 defer t.popScope();
187
188 const statements = t.ast.nodeStatements(module_node);
189 for (statements) |statement| {
190 try t.previsitModuleStatement(statement);
191 }
192
193 const s = t.scope();
194
195 const temp = try t.gpa.alloc(Inst.Index, s.symbols.count() + 1);
196 defer t.gpa.free(temp);
197
198 {
199 // prevent concurrent access
200 s.symbols.lockPointers();
201 defer s.symbols.unlockPointers();
202 for (s.symbols.values(), 0..) |value, i| {
203 temp[i] = try t.transformFunction(decl, value.node);
204 }
205 }
206
207 const top_level_fn = try t.addRootInst(.{
208 .tag = .declaration,
209 .data = undefined,
210 });
211
212 temp[temp.len - 1] = top_level_fn;
213
214 const declarations: IndexList = .{
215 .start = @intCast(t.extra.list.items.len),
216 .len = @intCast(temp.len),
217 };
218 try t.extra.list.appendSlice(t.gpa, @ptrCast(temp));
219
220 const block = try t.addRootInst(.{
221 .tag = .block,
222 .data = undefined,
223 });
224
225 {
226 const scratch_top = t.scratch.items.len;
227 defer t.scratch.shrinkRetainingCapacity(scratch_top);
228
229 for (statements) |statement| {
230 try t.transformModuleStatement(decl, statement);
231 }
232
233 t.getInstData(block).* = EncodedData.encode(.block, .{
234 .src_node = decl.nodeOffset(module_node),
235 .extra_index = try t.extra.append(t.gpa, IndexList, try t.scratchToIndexList(scratch_top)),
236 });
237 }
238
239 const decl_extra, var trail = try t.extra.appendTrailing(t.gpa, Inst.Decl, .{
240 .kind = .function,
241 .name = NullTerminatedString.empty,
242 .hash = .{0} ** 16,
243 .instruction_count = @as(u32, @intCast(t.insts.len)) - @intFromEnum(top_level_fn),
244 }, dod.extraSizeOf(Inst.DeclFn));
245
246 trail.writeStruct(Inst.DeclFn, .{
247 .params = .{ .start = 0, .len = 0 },
248 .return_type = Inst.Ref.void_type,
249 .body = block.toRef(),
250 });
251
252 t.getInstData(top_level_fn).* = EncodedData.encode(.declaration, .{
253 .src_node = decl.nodeOffset(module_node),
254 .extra_index = decl_extra,
255 });
256
257 t.getInstData(module).* = EncodedData.encode(.module, .{
258 .src_node = 0,
259 .extra_index = try t.extra.append(t.gpa, IndexList, declarations),
260 });
261
262 return module;
263}
264
265fn previsitModuleStatement(t: *Transform, node: NodeIndex) !void {
266 const trace = tracer.trace(@src());
267 defer trace.end();
268
269 const tag = t.ast.nodes.items(.tag)[node];
270 switch (tag) {
271 .fn_decl => {
272 const fn_decl = t.ast.fullFnDecl(node);
273 const gop = try t.scope().symbols.getOrPut(t.gpa, fn_decl.id);
274 if (gop.found_existing) {
275 return error.DuplicateFunction;
276 }
277 gop.value_ptr.* = .{
278 .node = node,
279 .data = .declaration,
280 };
281 },
282 else => {},
283 }
284}
285
286fn transformModuleStatement(t: *Transform, decl: BlockContext, node: NodeIndex) !void {
287 const trace = tracer.trace(@src());
288 defer trace.end();
289
290 const tag = t.ast.nodes.items(.tag)[node];
291 switch (tag) {
292 .root,
293 .param_decl,
294 => unreachable,
295
296 .fn_decl => {}, // functions analyzed separately
297
298 else => _ = try t.transformBlockStatement(decl, node, false),
299 }
300}
301
302fn transformFunction(t: *Transform, outer_decl: BlockContext, node: NodeIndex) !Inst.Index {
303 const trace = tracer.trace(@src());
304 defer trace.end();
305
306 const decl = outer_decl.declOffset(node);
307 const fn_decl = t.ast.fullFnDecl(node);
308
309 const func = try t.addRootInst(.{
310 .tag = .declaration,
311 .data = undefined,
312 });
313
314 try t.pushScope();
315 defer t.popScope();
316
317 const param_list = brk: {
318 const scratch_top = t.scratch.items.len;
319 defer t.scratch.shrinkRetainingCapacity(scratch_top);
320
321 for (fn_decl.params) |param| {
322 try t.addScratch(Inst.Ref, try t.transformParamDecl(decl, param));
323 }
324
325 break :brk try t.scratchToRefList(scratch_top);
326 };
327
328 const return_type = if (fn_decl.return_type) |return_type_node|
329 try t.transformExpressionToBlock(decl, return_type_node, false)
330 else
331 Inst.Ref.void_type;
332
333 const body = try t.transformExpressionToBlock(decl, fn_decl.body, true);
334
335 const decl_extra = try t.extra.append(t.gpa, Inst.Decl, .{
336 .kind = .function,
337 .name = try t.addNullTerminatedString(fn_decl.id),
338 .hash = .{0} ** 16,
339 .instruction_count = @as(u32, @intCast(t.insts.len)) - @intFromEnum(func),
340 });
341
342 _ = try t.extra.append(t.gpa, Inst.DeclFn, .{
343 .params = param_list,
344 .return_type = return_type,
345 .body = body,
346 });
347
348 t.getInstData(func).* = EncodedData.encode(.declaration, .{
349 .src_node = 0,
350 .extra_index = decl_extra,
351 });
352
353 return func;
354}
355
356const TransformFunctionCtx = struct {
357 param_list: RefList,
358};
359
360/// Converts expressions into blocks with a break. This is important
361/// for places where a block is not allowed, such as a parameter list.
362/// This is also used for the function body, even though it is a block.
363/// Simple refs turn into just the ref
364fn transformExpressionToBlock(
365 t: *Transform,
366 decl: BlockContext,
367 node: NodeIndex,
368 comptime is_function: bool,
369) anyerror!Inst.Ref {
370 const trace = tracer.trace(@src());
371 defer trace.end();
372
373 const node_tag = t.ast.nodes.items(.tag)[node];
374 switch (node_tag) {
375 .id => {
376 // Lower builtin ids without using a block
377 const id = t.ast.basicNodeSlice(node);
378 if (BuiltInIds.get(id)) |ref| {
379 return ref;
380 }
381 },
382
383 else => {},
384 }
385
386 var block: ?Inst.Index = null;
387
388 const scratch_top = t.scratch.items.len;
389 defer t.scratch.shrinkRetainingCapacity(scratch_top);
390
391 if (is_function) {
392 const s = t.scopes.getLast();
393 for (s.symbols.values(), 0..) |*param, i| {
394 assert(param.data == .param);
395 const ref = try t.addInstRef(.{
396 .tag = .param,
397 .data = EncodedData.encode(.param, .{
398 .src_node = decl.nodeOffset(node),
399 .data = @intCast(i),
400 }),
401 });
402 param.data = .{ .immutable = ref };
403 }
404 }
405
406 const expr = try t.transformExpression(
407 if (is_function)
408 decl.withReturn(&block)
409 else
410 decl,
411 node,
412 );
413
414 // if there are no new instructions, we dont have to create a block context
415 if (scratch_top == t.scratch.items.len) {
416 const result = expr orelse unreachable; // null means we added break or return
417
418 // TODO: evaluate if `if` blocks would be fine in sema with instruction refs
419 // with this extra if check here, we filter out non-blocks.
420 if (result.toIndex() == null) {
421 return result;
422 }
423 }
424
425 if (expr) |result| {
426 if (result.toIndex()) |index| {
427 if (t.insts.items(.tag)[@intFromEnum(index)] == .block) {
428 return result;
429 }
430 }
431 }
432
433 const ret_block = if (block) |b|
434 b
435 else
436 try t.addRootInst(.{
437 .tag = .block,
438 .data = undefined,
439 });
440
441 if (expr) |result| {
442 _ = try t.addInstRef(.{
443 .tag = .@"break",
444 .data = EncodedData.encode(.@"break", .{
445 .src_node = decl.nodeOffset(node),
446 .extra_index = try t.extra.append(t.gpa, Inst.Break, .{
447 .block = ret_block,
448 .operand = result,
449 }),
450 }),
451 });
452 }
453
454 const content = try t.scratchToIndexList(scratch_top);
455
456 t.getInstData(ret_block).* = EncodedData.encode(.block, .{
457 .src_node = decl.nodeOffset(node),
458 .extra_index = try t.extra.append(t.gpa, IndexList, content),
459 });
460
461 return ret_block.toRef();
462}
463
464fn transformParamDecl(t: *Transform, decl: BlockContext, param: NodeIndex) !Inst.Ref {
465 const trace = tracer.trace(@src());
466 defer trace.end();
467
468 const param_decl = t.ast.fullParamDecl(param);
469 const value = try t.transformExpressionToBlock(decl, param_decl.type, false);
470
471 // add `sym` into the scope so that the next param's type
472 // can use this param's value, if known at compile time.
473 const sym = try t.scope().symbols.getOrPut(t.gpa, param_decl.id);
474 if (sym.found_existing) {
475 return error.DuplicateParam;
476 }
477
478 sym.value_ptr.* = .{
479 .node = param,
480 .data = .param,
481 };
482
483 return value;
484}
485
486const Builtin = enum {
487 js,
488 import,
489 fetch,
490 compile_error,
491
492 const map = std.ComptimeStringMap(Builtin, .{
493 .{ "js", .js },
494 .{ "import", .import },
495 .{ "fetch", .fetch },
496 .{ "compileError", .compile_error },
497 });
498
499 fn builtin_js(t: *Transform, decl: BlockContext, args: []const NodeIndex) !?Inst.Ref {
500 const trace = tracer.trace(@src());
501 defer trace.end();
502
503 if (args.len != 1) {
504 return error.InvalidBuiltin;
505 }
506
507 const source = try t.transformExpression(decl, args[0]) orelse return null;
508
509 return try t.addInstRef(.{
510 .tag = .js,
511 .data = EncodedData.encode(.js, .{
512 .src_node = decl.nodeOffset(args[0]),
513 .operand = source,
514 }),
515 });
516 }
517};
518
519/// This is the primary transformation function: converting an AST expression into instructions.
520/// `null` represents a `return` or `break`, aka this branch of code is dead.
521fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) anyerror!?Inst.Ref {
522 const node_tag = t.ast.nodes.items(.tag)[node];
523
524 const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)});
525 defer trace.end();
526
527 switch (node_tag) {
528 .root,
529 .param_decl,
530 .var_decl,
531 .fn_decl,
532 => unreachable, // not an expression
533
534 .id => {
535 const id = t.ast.basicNodeSlice(node);
536 if (BuiltInIds.get(id)) |ref| {
537 return ref;
538 }
539
540 if (mem.eql(u8, id, "_")) {
541 return error.InvalidDiscard;
542 }
543
544 if (mem.startsWith(u8, id, "@")) {
545 @panic("TODO: string identifiers");
546 }
547
548 const lookup = t.lookupId(id) orelse {
549 std.debug.print("lookupId failed for {s}\n", .{id});
550 return error.UnknownIdentifier;
551 };
552
553 return switch (lookup.data) {
554 .declaration => try t.addInstRef(.{
555 .tag = .load_decl,
556 .data = EncodedData.encode(.load_decl, .{
557 .src_node = decl.nodeOffset(node),
558 .string = try t.addNullTerminatedString(id),
559 }),
560 }),
561 .mutable,
562 .immutable,
563 => |ref| ref,
564 .param => unreachable, //unresolved param
565 };
566 },
567
568 inline .number_int, .number_float => |tag| {
569 const text = t.ast.basicNodeSlice(node);
570 const value = switch (tag) {
571 .number_int => std.fmt.parseFloat(f64, text) catch @panic("TODO: write my own float parser"),
572 .number_float => std.fmt.parseFloat(f64, text) catch @panic("TODO: write my own float parser"),
573 else => comptime unreachable,
574 };
575 return try t.addInstRef(.{
576 .tag = .number,
577 .data = EncodedData.encode(.number, .{
578 .src_node = decl.nodeOffset(node),
579 .extra_index = try t.extra.append(t.gpa, Float, Float.encode(value)),
580 }),
581 });
582 },
583
584 .string => {
585 const str = t.ast.basicNodeSlice(node);
586 const sliced = try t.addSlicedString(str[1 .. str.len - 1]);
587 return try t.addInstRef(.{
588 .tag = .string,
589 .data = EncodedData.encode(.string, .{
590 .src_node = decl.nodeOffset(node),
591 .extra_index = try t.extra.append(t.gpa, SlicedString, sliced),
592 }),
593 });
594 },
595
596 .assign => {
597 const ast_bin = t.ast.nodeData(node).bin;
598 const lhs = try t.transformExpression(decl, ast_bin.lhs) orelse return null;
599 const rhs = try t.transformExpression(decl, ast_bin.rhs) orelse return null;
600 const assign_inst = try t.addInstRef(.{
601 .tag = .store,
602 .data = EncodedData.encode(.store, .{
603 .src_node = decl.nodeOffset(node),
604 .extra_index = try t.extra.append(t.gpa, Inst.Bin, .{
605 .lhs = lhs,
606 .rhs = rhs,
607 }),
608 }),
609 });
610 return assign_inst;
611 },
612
613 inline .add,
614 .sub,
615 .mul,
616 .div,
617 .rem,
618 .pow,
619 .@"and",
620 .@"or",
621 .bit_and,
622 .bit_or,
623 .bit_xor,
624 .shl,
625 .shr,
626 .eq,
627 .ne,
628 .lt,
629 .le,
630 .gt,
631 .ge,
632 .add_assign,
633 .sub_assign,
634 .mul_assign,
635 .div_assign,
636 .rem_assign,
637 .pow_assign,
638 .and_assign,
639 .or_assign,
640 .bit_and_assign,
641 .bit_or_assign,
642 .bit_xor_assign,
643 .shl_assign,
644 .shr_assign,
645 => |tag| {
646 // This code path implements binary operations, as well as the
647 // compound compute + assign operation.
648 const is_assign = comptime tag.isAssignBin();
649 const teg_tag = @field(
650 Teg.Inst.Tag,
651 if (is_assign)
652 @tagName(tag)[0 .. @tagName(tag).len - "_assign".len]
653 else
654 @tagName(tag),
655 );
656 const ast_bin = t.ast.nodeData(node).bin;
657 const lhs = try t.transformExpression(decl, ast_bin.lhs) orelse return null;
658 const rhs = try t.transformExpression(decl, ast_bin.rhs) orelse return null;
659
660 const compute_inst = try t.addInstRef(.{
661 .tag = teg_tag,
662 .data = EncodedData.encode(teg_tag, .{
663 .src_node = decl.nodeOffset(node),
664 .extra_index = try t.extra.append(t.gpa, Inst.Bin, .{
665 .lhs = lhs,
666 .rhs = rhs,
667 }),
668 }),
669 });
670
671 if (is_assign) {
672 const assign_inst = try t.addInstRef(.{
673 .tag = .store,
674 .data = EncodedData.encode(.store, .{
675 .src_node = decl.nodeOffset(node),
676 .extra_index = try t.extra.append(t.gpa, Inst.Bin, .{
677 .lhs = lhs,
678 .rhs = compute_inst,
679 }),
680 }),
681 });
682 return assign_inst;
683 }
684
685 return compute_inst;
686 },
687
688 inline .negate,
689 .bool_not,
690 .bit_not,
691 => |tag| {
692 const ast_un = t.ast.nodeData(node).un;
693 const teg_tag = @field(Teg.Inst.Tag, @tagName(tag));
694
695 const operand = try t.transformExpression(decl, ast_un.expr) orelse
696 return null;
697
698 return try t.addInstRef(.{
699 .tag = teg_tag,
700 .data = EncodedData.encode(teg_tag, .{
701 .src_node = decl.nodeOffset(node),
702 .operand = operand,
703 }),
704 });
705 },
706
707 .label => {
708 const expr = t.ast.nodeData(node).label.expr;
709 const label = t.ast.tokenSlice(t.ast.nodes.items(.main_token)[node]);
710
711 var target: BlockContext.BreakTarget = .{
712 .parent = decl.break_target,
713 .label = label,
714 .is_loop = false,
715 };
716 const inner_ctx = decl.withBreak(&target);
717
718 const scratch_top = t.scratch.items.len;
719
720 const inner_expr = try t.transformExpression(inner_ctx, expr);
721
722 // if there are no new instructions, we dont have to create a block context
723 if (scratch_top == t.scratch.items.len)
724 return inner_expr orelse unreachable; // null means we added break or return
725
726 if (target.block_index) |block| {
727 if (inner_expr) |result| {
728 _ = try t.addInstRef(.{
729 .tag = .@"break",
730 .data = EncodedData.encode(.@"break", .{
731 .src_node = decl.nodeOffset(node),
732 .extra_index = try t.extra.append(t.gpa, Inst.Break, .{
733 .block = block,
734 .operand = result,
735 }),
736 }),
737 });
738 }
739
740 const content = try t.scratchToIndexList(scratch_top);
741 t.scratch.shrinkRetainingCapacity(scratch_top);
742
743 t.getInstData(block).* = EncodedData.encode(.block, .{
744 .src_node = decl.nodeOffset(node),
745 .extra_index = try t.extra.append(t.gpa, IndexList, content),
746 });
747
748 try t.addScratch(Inst.Index, block);
749
750 return block.toRef();
751 } else {
752 std.debug.print("Warn, unused label {s}", .{label});
753 }
754
755 return inner_expr;
756 },
757
758 inline .block_void,
759 .block_implicit_return,
760 => |tag| {
761 const statements = t.ast.nodeStatements(node);
762
763 var last: Inst.Ref = .void_value;
764 for (statements, 1..) |statement, i| {
765 const is_last = i == statements.len;
766 last = try t.transformBlockStatement(
767 decl,
768 statement,
769 tag == .block_implicit_return and is_last,
770 ) orelse {
771 return null;
772 };
773 }
774
775 return if (tag == .block_implicit_return) last else .void_value;
776 },
777
778 .builtin_call,
779 => {
780 const builtin = Builtin.map.get(t.ast.basicNodeSlice(node)[1..]) orelse {
781 return error.InvalidBuiltin;
782 };
783
784 const base = t.ast.nodeData(node).list;
785 const args = base.slice(&t.ast);
786
787 return switch (builtin) {
788 .js => Builtin.builtin_js(t, decl, args),
789 // .import => Builtin.import(t, decl, args),
790 // .fetch => Builtin.fetch(t, decl, args),
791 // .compile_error => Builtin.compile_error(t, decl, args),
792 else => @panic("TODO"),
793 };
794 },
795
796 .call => {
797 const call = t.ast.nodeData(node).call;
798 const args = t.ast.extraData(call.extra, Ast.Call).args.slice(&t.ast);
799
800 const base = try t.transformExpression(decl, call.base) orelse return null;
801
802 const args_list: RefList = brk: {
803 const args_top = t.call_args.items.len;
804 defer t.call_args.shrinkRetainingCapacity(args_top);
805
806 for (args) |arg| {
807 try t.call_args.append(
808 t.gpa,
809 try t.transformExpression(decl, arg) orelse
810 return null,
811 );
812 }
813
814 const items = t.call_args.items[args_top..];
815 const extra_top = t.extra.list.items.len;
816 try t.extra.list.appendSlice(t.gpa, @ptrCast(items));
817
818 break :brk .{
819 .start = @intCast(extra_top),
820 .len = @intCast(items.len),
821 };
822 };
823
824 const call_inst = try t.addInstRef(.{
825 .tag = .call,
826 .data = EncodedData.encode(.call, .{
827 .src_node = decl.nodeOffset(node),
828 .extra_index = try t.extra.append(t.gpa, Inst.Call, .{
829 .base = base,
830 .args = args_list,
831 }),
832 }),
833 });
834
835 return call_inst;
836 },
837
838 .field => {
839 const field = t.ast.nodeData(node).field;
840
841 const base = try t.transformExpression(decl, field.base) orelse return null;
842
843 // TODO: handle identifier escapes
844 const id = t.ast.tokenSlice(field.field_token);
845 const prop_name_str = try t.addSlicedString(id);
846
847 const string_ref = try t.addInstRef(.{
848 .tag = .string,
849 .data = EncodedData.encode(.string, .{
850 .src_node = decl.nodeOffset(node), // TODO: this is incorrect, there is no node
851 .extra_index = try t.extra.append(t.gpa, SlicedString, prop_name_str),
852 }),
853 });
854
855 return try t.addInstRef(.{
856 .tag = .field,
857 .data = EncodedData.encode(.field, .{
858 .src_node = decl.nodeOffset(node),
859 .extra_index = try t.extra.append(t.gpa, Inst.Bin, .{
860 .lhs = base,
861 .rhs = string_ref,
862 }),
863 }),
864 });
865 },
866
867 .@"if" => {
868 const if_node = t.ast.nodeData(node).@"if";
869 const if_data = t.ast.extraData(if_node.extra, Ast.If);
870
871 const cond = try t.transformExpression(decl, if_node.cond) orelse
872 return null;
873
874 const yes = try t.transformExpressionToBlock(
875 decl,
876 if_data.yes,
877 false,
878 );
879 const no = if (if_data.no.unwrap()) |no|
880 try t.transformExpressionToBlock(
881 decl,
882 no,
883 false,
884 )
885 else
886 Inst.Ref.void_value;
887
888 if (yes == .void_value and no == .void_value) {
889 // when both branches do nothing, we can omit the branch,
890 // however we still need to type-check the condition operand
891 _ = try t.addInstRef(.{
892 .tag = .ensure_bool,
893 .data = EncodedData.encode(.ensure_bool, .{
894 .src_node = decl.nodeOffset(node),
895 .operand = cond,
896 }),
897 });
898 return .void_value;
899 }
900
901 return try t.addInstRef(.{
902 .tag = .@"if",
903 .data = EncodedData.encode(.@"if", .{
904 .src_node = decl.nodeOffset(node),
905 .extra_index = try t.extra.append(t.gpa, Inst.If, .{
906 .cond = cond,
907 .yes = yes,
908 .no = no,
909 }),
910 }),
911 });
912 },
913
914 .@"return" => {
915 const ret_node = t.ast.nodeData(node).@"return";
916
917 const ret_block_ptr = decl.return_target orelse {
918 return error.InvalidReturn;
919 };
920
921 const ret_block = if (ret_block_ptr.*) |block|
922 block
923 else append_block: {
924 const block = try t.addRootInst(.{
925 .tag = .block,
926 .data = undefined,
927 });
928 ret_block_ptr.* = block;
929 break :append_block block;
930 };
931
932 const operand: Inst.Ref = if (ret_node.expr.unwrap()) |expr|
933 try t.transformExpression(decl.withReturn(null), expr) orelse
934 return null
935 else
936 .void_value;
937
938 _ = try t.addInstRef(.{
939 .tag = .@"break",
940 .data = EncodedData.encode(.@"break", .{
941 .src_node = decl.nodeOffset(node),
942 .extra_index = try t.extra.append(t.gpa, Inst.Break, .{
943 .block = ret_block,
944 .operand = operand,
945 }),
946 }),
947 });
948
949 return null; // control flow ends
950 },
951
952 .@"break" => {
953 const brk_node = t.ast.nodeData(node).@"break";
954 const label = t.ast.tokenSlice(
955 brk_node.label.unwrap() orelse
956 @panic("TODO: unlabelled breaks"),
957 );
958
959 const block = try decl.findAndMarkBreak(t, label) orelse {
960 return error.UnknownBreakTarget;
961 };
962
963 const operand: Inst.Ref = if (brk_node.value.unwrap()) |expr|
964 try t.transformExpression(decl.withReturn(null), expr) orelse
965 return null
966 else
967 .void_value;
968
969 _ = try t.addInstRef(.{
970 .tag = .@"break",
971 .data = EncodedData.encode(.@"break", .{
972 .src_node = decl.nodeOffset(node),
973 .extra_index = try t.extra.append(t.gpa, Inst.Break, .{
974 .block = block,
975 .operand = operand,
976 }),
977 }),
978 });
979
980 return null; // control flow ends
981 },
982
983 else => |tag| std.debug.panic("TODO: expr {s}", .{@tagName(tag)}),
984 }
985}
986
987fn transformBlockStatement(
988 t: *Transform,
989 decl: BlockContext,
990 node: NodeIndex,
991 is_implicit_return_expression: bool,
992) !?Inst.Ref {
993 const node_tag = t.ast.nodes.items(.tag)[node];
994
995 const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)});
996 defer trace.end();
997
998 switch (node_tag) {
999 .root => unreachable, // cannot appear in a block
1000 .param_decl => unreachable, // cannot appear in a block
1001
1002 .var_decl => {
1003 const var_decl = t.ast.fullVarDecl(node);
1004 const type_ref = if (var_decl.type) |data_type|
1005 try t.transformExpression(decl, data_type)
1006 else
1007 null;
1008 const data_ref = try t.transformExpression(decl, var_decl.init) orelse
1009 return null;
1010 const data_coerced_ref = if (type_ref) |ref|
1011 try t.addInstRef(.{
1012 .tag = .as,
1013 .data = EncodedData.encode(.as, .{
1014 .src_node = decl.nodeOffset(node),
1015 .extra_index = try t.extra.append(t.gpa, Inst.As, .{
1016 .type = ref,
1017 .operand = data_ref,
1018 }),
1019 }),
1020 })
1021 else
1022 data_ref;
1023
1024 const mem_ref = if (var_decl.is_const)
1025 data_coerced_ref
1026 else
1027 try t.addInstRef(.{
1028 .tag = .memory,
1029 .data = EncodedData.encode(.memory, .{
1030 .src_node = decl.nodeOffset(node),
1031 .operand = data_coerced_ref,
1032 }),
1033 });
1034
1035 try t.scope().symbols.put(t.gpa, var_decl.id, .{
1036 .node = node,
1037 .data = switch (var_decl.is_const) {
1038 true => .{ .immutable = mem_ref },
1039 false => .{ .mutable = mem_ref },
1040 },
1041 });
1042
1043 return mem_ref;
1044 },
1045
1046 .fn_decl,
1047 => |tag| {
1048 std.debug.panic("TODO: block statement {s}", .{@tagName(tag)});
1049 },
1050
1051 .assign,
1052 => {
1053 const bin = t.ast.nodeData(node).bin;
1054 const lhs = bin.lhs;
1055
1056 if (t.ast.nodeTag(lhs) == .id and
1057 mem.eql(u8, t.ast.basicNodeSlice(lhs), "_"))
1058 {
1059 // discard, instead of appending an .ensure_void check,
1060 // we will add a .discard instruction instead.
1061 // this will emit a warning if given void.
1062 const result = try t.transformExpression(decl, bin.rhs) orelse
1063 return null;
1064 return try t.addInstRef(.{
1065 .tag = .discard,
1066 .data = EncodedData.encode(.discard, .{
1067 .src_node = decl.nodeOffset(node),
1068 .operand = result,
1069 }),
1070 });
1071 } else {
1072 // process this assignment as usual, do not append .ensure_void
1073 return try t.transformExpression(decl, node);
1074 }
1075 },
1076
1077 .add_assign,
1078 .sub_assign,
1079 .mul_assign,
1080 .div_assign,
1081 .rem_assign,
1082 .pow_assign,
1083 .and_assign,
1084 .or_assign,
1085 .bit_and_assign,
1086 .bit_or_assign,
1087 .bit_xor_assign,
1088 .shl_assign,
1089 .shr_assign,
1090 => {
1091 // do not append a .ensure_void check
1092 return try t.transformExpression(decl, node);
1093 },
1094
1095 else => {
1096 const result = try t.transformExpression(decl, node) orelse
1097 return null;
1098
1099 if (!is_implicit_return_expression and result != .void_value) {
1100 return try t.addInstRef(.{
1101 .tag = .ensure_void,
1102 .data = EncodedData.encode(.ensure_void, .{
1103 .src_node = decl.nodeOffset(node),
1104 .operand = result,
1105 }),
1106 });
1107 }
1108
1109 return result;
1110 },
1111 }
1112}
1113
1114// ---
1115
1116fn addRootInst(t: *Transform, inst: Inst) !Inst.Index {
1117 try t.insts.append(t.gpa, inst);
1118 return @enumFromInt(t.insts.len - 1);
1119}
1120
1121fn addInstRef(t: *Transform, inst: Inst) !Inst.Ref {
1122 try t.scratch.ensureUnusedCapacity(t.gpa, 1);
1123 try t.insts.append(t.gpa, inst);
1124 const index: Inst.Index = @enumFromInt(t.insts.len - 1);
1125 t.scratch.appendAssumeCapacity(@intFromEnum(index));
1126 return index.toRef();
1127}
1128
1129fn pushScope(t: *Transform) !void {
1130 try t.scopes.append(t.gpa, .{});
1131}
1132
1133fn popScope(t: *Transform) void {
1134 t.scope().deinit(t.gpa);
1135 t.scopes.items[t.scopes.items.len - 1] = undefined;
1136 t.scopes.items.len -= 1;
1137}
1138
1139fn scope(t: *Transform) *Scope {
1140 return &t.scopes.items[t.scopes.items.len - 1];
1141}
1142
1143fn getInstData(t: *Transform, index: Inst.Index) *EncodedData {
1144 return &t.insts.items(.data)[@intFromEnum(index)];
1145}
1146
1147fn addScratch(t: *Transform, comptime T: type, value: T) !void {
1148 try t.scratch.append(t.gpa, switch (T) {
1149 Inst.Ref, Inst.Index => @intFromEnum(value),
1150 else => @compileError("addScratch not implemented for " ++ @typeName(T)),
1151 });
1152}
1153
1154fn scratchToRefList(t: *Transform, scratch_top: usize) !RefList {
1155 const items = t.scratch.items[scratch_top..];
1156 const extra_top = t.extra.list.items.len;
1157 try t.extra.list.appendSlice(t.gpa, t.scratch.items[scratch_top..]);
1158 return .{
1159 .start = @intCast(extra_top),
1160 .len = @intCast(items.len),
1161 };
1162}
1163
1164fn scratchToIndexList(t: *Transform, scratch_top: usize) !IndexList {
1165 const items = t.scratch.items[scratch_top..];
1166 const extra_top = t.extra.list.items.len;
1167 try t.extra.list.appendSlice(t.gpa, items);
1168 return .{
1169 .start = @intCast(extra_top),
1170 .len = @intCast(items.len),
1171 };
1172}
1173
1174fn lookupId(t: *Transform, id: []const u8) ?Scope.Symbol {
1175 var i = t.scopes.items.len;
1176 while (i > 0) {
1177 i -= 1;
1178 if (t.scopes.items[i].symbols.get(id)) |sym| {
1179 return sym;
1180 }
1181 }
1182 return null;
1183}
1184
1185fn addNullTerminatedString(t: *Transform, str: []const u8) !NullTerminatedString {
1186 if (str.len == 0) {
1187 return .empty;
1188 }
1189 // loop through strings trying to find an existing match, and dedupe memory
1190 var pos: usize = 0;
1191 while (mem.indexOfScalarPos(u8, str, pos, 0)) |i| : (pos = i + 1) {
1192 const start_index = i - str.len;
1193 if (mem.eql(u8, t.string_bytes.items[start_index..][0..str.len], str)) {
1194 const string: NullTerminatedString = @enumFromInt(pos);
1195 assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
1196 return string;
1197 }
1198 }
1199
1200 const string: NullTerminatedString = @enumFromInt(t.string_bytes.items.len);
1201 try t.string_bytes.ensureUnusedCapacity(t.gpa, str.len + 1);
1202 t.string_bytes.appendSliceAssumeCapacity(str);
1203 t.string_bytes.appendAssumeCapacity(0);
1204 assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
1205 return string;
1206}
1207
1208fn addSlicedString(t: *Transform, str: []const u8) !SlicedString {
1209 if (str.len == 0) {
1210 return .{ .start = 0, .len = 0 };
1211 }
1212
1213 // loop through strings trying to find an existing match, and dedupe memory
1214 var pos: usize = 0;
1215 while (mem.indexOfScalarPos(u8, str, pos, 0)) |i| : (pos = i + 1) {
1216 const start_index = i - str.len;
1217 if (mem.eql(u8, t.string_bytes.items[start_index..][0..str.len], str)) {
1218 const string: SlicedString = .{
1219 .start = @intCast(start_index),
1220 .len = @intCast(str.len),
1221 };
1222 assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
1223 return string;
1224 }
1225 }
1226
1227 const string: SlicedString = .{
1228 .start = @intCast(t.string_bytes.items.len),
1229 .len = @intCast(str.len),
1230 };
1231 try t.string_bytes.ensureUnusedCapacity(t.gpa, str.len);
1232 t.string_bytes.appendSliceAssumeCapacity(str);
1233 assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
1234 return string;
1235}
1236
1237const Transform = @This();
1238const std = @import("std");
1239const mem = std.mem;
1240const Allocator = mem.Allocator;
1241const assert = std.debug.assert;
1242
1243const Teg = @import("Teg.zig");
1244const Inst = Teg.Inst;
1245const NullTerminatedString = Teg.NullTerminatedString;
1246const SlicedString = Teg.SlicedString;
1247const RefList = Teg.RefList;
1248const IndexList = Teg.IndexList;
1249const EncodedData = Teg.EncodedData;
1250const ExtraIndex = Teg.ExtraIndex;
1251
1252const dod = @import("dod.zig");
1253const ExtraStore = dod.ExtraStore(.teg, .@"var");
1254const Float = dod.Float;
1255
1256const Tokenizer = @import("Tokenizer.zig");
1257const Token = Tokenizer.Token;
1258const ByteOffset = Tokenizer.ByteOffset;
1259
1260const Ast = @import("Ast.zig");
1261const Unused = Ast.Unused;
1262const Node = Ast.Node;
1263const Data = Node.Data;
1264const SubList = Ast.SubList;
1265const NodeIndex = Ast.NodeIndex;
1266const TokenIndex = Ast.TokenIndex;
1267const OptionalNodeIndex = Ast.OptionalNodeIndex;
1268const OptionalTokenIndex = Ast.OptionalTokenIndex;
1269const OptionalExtraIndex = Ast.OptionalExtraIndex;
1270const optionalTokenIndex = OptionalTokenIndex.init;
1271const optionalNodeIndex = OptionalNodeIndex.init;
1272const optionalExtraIndex = OptionalExtraIndex.init;
1273
1274const InternPool = @import("InternPool.zig");
1275
1276const tracer = @import("tracer");
src/compiler/Value.zig created+89
......@@ -0,0 +1,89 @@
1ip_index: InternPool.Index,
2
3pub fn fromInterned(i: InternPool.Index) Value {
4 assert(i != .none);
5 return .{ .ip_index = i };
6}
7
8pub fn toIntern(val: Value) InternPool.Index {
9 assert(val.ip_index != .none);
10 return val.ip_index;
11}
12
13pub fn toKey(val: Value, ip: *const InternPool) InternPool.Key {
14 return val.toIntern().toKey(ip);
15}
16
17pub fn asBool(val: Value) i32 {
18 return switch (val.toIntern()) {
19 .bool_true => true,
20 .bool_false => false,
21 else => unreachable,
22 };
23}
24
25pub fn asString(val: Value, ip: *InternPool) InternPool.String.WithLen {
26 const index = val.toIntern();
27 return ip.items.items(.data)[@intFromEnum(index)].decode(
28 .string,
29 ip.items.items(.tag)[@intFromEnum(index)],
30 )
31 .get(ip);
32}
33
34pub fn @"type"(val: Value, ip: *InternPool) Value {
35 std.debug.print("typeOfValue {any}\n", .{val.toKey(ip)});
36 const v: InternPool.Index = switch (val.toKey(ip)) {
37 .simple_type, .type_function => .type_type,
38
39 .int32 => .int32_type,
40 .float => .float_type,
41 .string => .string_type,
42 .simple_value => |v| switch (v) {
43 .true, .false => .bool_type,
44 .void => .void_type,
45 },
46 .func => |f| f.ty,
47 };
48 return Value.fromInterned(v);
49}
50
51/// used for validating ensure_void instructions
52pub fn isEnsureVoid(val: Value, ip: *InternPool) bool {
53 const intern = val.toIntern();
54 const i = @intFromEnum(intern);
55
56 return switch (ip.items.items(.tag)[i]) {
57 .simple_type => switch (ip.items.items(.data)[i].decode(.simple_type, .simple_type)) {
58 .@"opaque", .void => true,
59 else => true,
60 },
61 else => false,
62 };
63}
64
65const Value = @This();
66
67const std = @import("std");
68const assert = std.debug.assert;
69
70pub const InternPool = @import("InternPool.zig");
71
72pub fn fmt(v: Value, ip: *InternPool) Format {
73 return .{ .value = v, .ip = ip };
74}
75
76pub const Format = struct {
77 value: Value,
78 ip: *InternPool,
79
80 pub fn format(
81 self: Format,
82 comptime unused_fmt_str: []const u8,
83 _: std.fmt.FormatOptions,
84 writer: anytype,
85 ) !void {
86 comptime assert(unused_fmt_str.len == 0);
87 try writer.print("{any}", .{self.value.toKey(self.ip)});
88 }
89};
src/compiler/dod.zig created+382
......@@ -0,0 +1,382 @@
1const extra = @This();
2
3pub const Mutability = enum { @"const", @"var" };
4
5fn setAsU32(items: []u32, comptime T: type, index: usize, payload: T) void {
6 const fields = std.meta.fields(@TypeOf(payload));
7 var i = index;
8 inline for (fields) |field| {
9 const value = @field(payload, field.name);
10 switch (field.type) {
11 u32 => items[i] = value,
12 i32 => items[i] = @bitCast(value),
13
14 [16]u8 => {
15 const u32s: [4]u32 = @bitCast(value);
16 items[i] = u32s[0];
17 items[i + 1] = u32s[1];
18 items[i + 2] = u32s[2];
19 items[i + 3] = u32s[3];
20 },
21
22 else => brk: {
23 const info = @typeInfo(field.type);
24 if (info == .Enum) {
25 items[i] = @intFromEnum(value);
26 break :brk;
27 }
28
29 const struct_data = info.Struct;
30 if (struct_data.layout == .@"packed") {
31 items[i] = @bitCast(value);
32 break :brk;
33 }
34
35 @call(.always_inline, setAsU32, .{ items, field.type, i, value });
36 },
37 }
38 i += extraSizeOf(field.type);
39 }
40}
41
42fn getAsU32(items: []const u32, comptime T: type, index: usize) T {
43 const fields = std.meta.fields(T);
44 var i = index;
45 var result: T = undefined;
46 inline for (fields) |field| {
47 @field(result, field.name) = switch (field.type) {
48 u32 => items[i],
49 i32 => @bitCast(items[i]),
50
51 [16]u8 => @bitCast(items[i..][0..4].*),
52
53 else => brk: {
54 const info = @typeInfo(field.type);
55 if (info == .Enum) {
56 break :brk @enumFromInt(items[i]);
57 }
58
59 const struct_data = info.Struct;
60 if (struct_data.layout == .@"packed") {
61 break :brk @bitCast(items[i]);
62 }
63
64 break :brk @call(
65 .always_inline,
66 getAsU32,
67 .{ items, field.type, i },
68 );
69 },
70 };
71 i += extraSizeOf(field.type);
72 }
73 return result;
74}
75
76pub const StoreKind = enum {
77 intern_pool,
78 teg,
79 air,
80 tfr,
81
82 pub fn resolveStoreConst(
83 comptime kind: StoreKind,
84 owner: anytype,
85 ) *const kind.ResolvedStore(@TypeOf(owner)) {
86 return &owner.extra;
87 }
88
89 pub fn resolveStoreMut(
90 comptime kind: StoreKind,
91 owner: anytype,
92 ) *kind.ResolvedStore(@TypeOf(owner)) {
93 return &owner.extra;
94 }
95
96 pub fn ResolvedStore(
97 comptime kind: StoreKind,
98 comptime Owner: type,
99 ) type {
100 const ptr = @typeInfo(Owner).Pointer;
101 assert(ptr.size == .One);
102 const mutability = brk: for (@typeInfo(ptr.child).Struct.fields) |field| {
103 if (std.mem.eql(u8, field.name, "extra")) {
104 break :brk field.type.mutability;
105 }
106 } else unreachable;
107 return ExtraStore(kind, mutability);
108 }
109};
110
111fn ExtraWrap(comptime kind: StoreKind) type {
112 return struct {
113 pub fn Index(comptime T: type) type {
114 return enum(u32) {
115 _,
116
117 // Reads the data from `store.extra` and returns it.
118 pub fn get(index: @This(), owner: anytype) T {
119 return kind.resolveStoreConst(owner).get(T, @intFromEnum(index));
120 }
121
122 /// Returns data with a trailing reader to read data stored immediately after
123 /// the main data structure. What is stored should be documented on `T`
124 pub fn getTrail(index: @This(), owner: anytype) struct {
125 T,
126 kind.ResolvedStore(@TypeOf(owner)).TrailReader,
127 } {
128 const advance = comptime extraSizeOf(T);
129 return .{
130 index.get(owner),
131 .{ .index = @intFromEnum(index) + advance },
132 };
133 }
134 };
135 }
136 };
137}
138
139pub fn EncodedData(comptime Tag: type, backing_type: type) type {
140 return enum(backing_type) {
141 _,
142
143 pub fn decode(data: @This(), comptime tag: Tag, proof: Tag) tag.Data() {
144 assert(proof == tag); // incorrect memory access
145 const Data = tag.Data();
146 const info = @typeInfo(Data);
147 if (info == .Enum) {
148 return @enumFromInt(@intFromEnum(data));
149 }
150 if (info == .Struct) {
151 assert(backing_type == u64);
152 var fields: [2]u32 = @bitCast(@intFromEnum(data));
153 return getAsU32(&fields, Data, 0);
154 }
155 return @bitCast(@intFromEnum(data));
156 }
157
158 pub fn encode(comptime tag: Tag, data: tag.Data()) @This() {
159 const Data = tag.Data();
160 const info = @typeInfo(Data);
161 if (info == .Enum) {
162 return @enumFromInt(@intFromEnum(data));
163 }
164 if (info == .Struct) {
165 assert(backing_type == u64);
166 var fields: [2]u32 = undefined;
167 setAsU32(&fields, Data, 0, data);
168 return @enumFromInt(@as(backing_type, @bitCast(fields)));
169 }
170 return @enumFromInt(@as(backing_type, @bitCast(data)));
171 }
172 };
173}
174
175/// The number of 'u32's this type consumes in `ip.extra`.
176pub fn extraSizeOf(comptime T: type) usize {
177 return switch (T) {
178 u32,
179 i32,
180 => return 1,
181
182 [16]u8 => return 4,
183
184 else => {
185 const info = @typeInfo(T);
186 if (info == .Enum) {
187 comptime assert(@sizeOf(info.Enum.tag_type) <= 32);
188 return 1;
189 }
190
191 const struct_data = info.Struct;
192 if (struct_data.layout == .@"packed") {
193 comptime assert(struct_data.backing_integer == u32);
194 return 1;
195 }
196
197 const fields = std.meta.fields(T);
198 var size: usize = 0;
199 inline for (fields) |field| {
200 size += extraSizeOf(field.type);
201 }
202 return size;
203 },
204 };
205}
206
207pub fn ExtraStore(comptime set_kind: StoreKind, comptime set_mutability: Mutability) type {
208 return struct {
209 list: List = .{},
210
211 pub const kind = set_kind;
212 pub const mutability = set_mutability;
213
214 const Store = @This();
215
216 const index_holder = ExtraWrap(kind);
217 pub const Index = index_holder.Index;
218
219 pub const List = switch (mutability) {
220 .@"const" => struct {
221 items: []const u32 = &.{},
222
223 pub fn deinit(slice: *@This(), gpa: Allocator) void {
224 gpa.free(slice.items);
225 }
226 },
227 .@"var" => std.ArrayListUnmanaged(u32),
228 };
229
230 pub fn deinit(store: *Store, gpa: Allocator) void {
231 store.list.deinit(gpa);
232 store.* = undefined;
233 }
234
235 pub const TrailReader = struct {
236 index: usize,
237
238 pub fn read(trail: *TrailReader, ip: *const Store, comptime T: type) T {
239 defer trail.index += comptime extraSizeOf(T);
240 return ip.get(T, trail.index);
241 }
242
243 pub fn readInternPoolIndexSlice(trail: *TrailReader, len: u32) @import("InternPool.zig").Index.Slice {
244 defer trail.index += len;
245 return .{
246 .start = @intCast(trail.index),
247 .len = len,
248 };
249 }
250 };
251
252 pub const TrailWriter = struct {
253 store: *Store,
254 leftover: usize,
255
256 pub fn init(store: *Store, len: usize) TrailWriter {
257 return .{ .store = store, .leftover = len };
258 }
259
260 pub fn end(w: *TrailWriter) void {
261 assert(w.leftover == 0);
262 }
263
264 pub fn writeU32(w: *TrailWriter, value: u32) void {
265 assert(w.leftover > 0);
266 w.ip.extra.appendAssumeCapacity(value);
267 w.leftover -= 1;
268 }
269
270 pub fn writeSlice(w: *TrailWriter, slice: anytype) void {
271 const Child = std.meta.Child(@TypeOf(slice));
272 const child = @typeInfo(Child);
273 comptime assert(Child == u32 or (child == .Enum and child.Enum.tag_type == u32));
274 assert(w.leftover >= slice.len);
275 w.store.list.appendSliceAssumeCapacity(@ptrCast(slice));
276 w.leftover -= slice.len;
277 }
278
279 pub fn writeStruct(w: *TrailWriter, comptime T: type, data: T) void {
280 _ = appendAssumeCapacity(w.store, T, data);
281 }
282 };
283
284 pub fn append(store: *Store, gpa: Allocator, comptime T: type, payload: T) Allocator.Error!Index(T) {
285 comptime assert(mutability == .@"var");
286 try store.list.ensureUnusedCapacity(gpa, extraSizeOf(@TypeOf(payload)));
287 return store.appendAssumeCapacity(T, payload);
288 }
289
290 pub fn appendAssumeCapacity(ip: *Store, comptime T: type, payload: T) Index(T) {
291 comptime assert(mutability == .@"var");
292 const extra_index: u32 = @intCast(ip.list.items.len);
293 ip.list.items.len += extraSizeOf(@TypeOf(payload));
294 ip.set(T, extra_index, payload);
295 return @enumFromInt(extra_index);
296 }
297
298 pub fn appendTrailing(
299 store: *Store,
300 gpa: Allocator,
301 comptime T: type,
302 payload: T,
303 trailing_len: usize,
304 ) Allocator.Error!struct { Index(T), TrailWriter } {
305 comptime assert(mutability == .@"var");
306 try store.list.ensureUnusedCapacity(gpa, extraSizeOf(@TypeOf(payload)) + trailing_len);
307 const extra_index: u32 = @intCast(store.list.items.len);
308 store.list.items.len += extraSizeOf(@TypeOf(payload));
309 store.set(T, extra_index, payload);
310 return .{
311 @enumFromInt(extra_index),
312 TrailWriter.init(store, trailing_len),
313 };
314 }
315
316 fn set(store: *Store, comptime T: type, index: usize, payload: T) void {
317 comptime assert(mutability == .@"var");
318 setAsU32(store.list.items, T, index, payload);
319 }
320
321 fn get(store: *const Store, comptime T: type, index: usize) T {
322 return getAsU32(store.list.items, T, index);
323 }
324
325 pub fn toOwnedStore(store: *Store, gpa: Allocator) !ExtraStore(set_kind, .@"const") {
326 comptime assert(mutability == .@"var");
327 return .{ .list = .{ .items = try store.list.toOwnedSlice(gpa) } };
328 }
329
330 pub fn constant(store: *const Store) ExtraStore(set_kind, .@"const") {
331 return .{ .list = .{ .items = store.list.items } };
332 }
333 };
334}
335
336pub const Float = struct {
337 low: u32,
338 high: u32,
339
340 pub const pure_nan = Float{ .low = 0x7f800000, .high = 0 };
341
342 pub fn purifyNaN(f: f64) f64 {
343 if (std.math.isNan(f)) {
344 return comptime Float.pure_nan.decode();
345 }
346 return f;
347 }
348
349 pub fn assertPure(f: f64) f64 {
350 if (std.math.isNan(f)) {
351 assert(Float.pure_nan.eql(encode(f)));
352 }
353 return f;
354 }
355
356 pub fn encode(decoded: f64) Float {
357 if (std.math.isNan(decoded)) {
358 return pure_nan;
359 }
360 const as_bytes: [2]u32 = @bitCast(decoded);
361 return .{
362 .low = as_bytes[0],
363 .high = as_bytes[1],
364 };
365 }
366
367 pub fn decode(encoded: Float) f64 {
368 const as_bytes: [2]u32 = .{
369 encoded.low,
370 encoded.high,
371 };
372 return @bitCast(as_bytes);
373 }
374
375 pub fn eql(a: Float, b: Float) bool {
376 return a.low == b.low and a.high == b.high;
377 }
378};
379
380const std = @import("std");
381const assert = std.debug.assert;
382const Allocator = std.mem.Allocator;
src/compiler/main.zig created+180
......@@ -0,0 +1,180 @@
1const std = @import("std");
2const Tokenizer = @import("Tokenizer.zig");
3const Parse = @import("Parse.zig");
4const Transform = @import("Transform.zig");
5const Comp = @import("Comp.zig");
6
7const T = @import("tracer");
8pub const tracer_impl = @field(T, "none");
9
10pub fn main() !void {
11 std.debug.print("\r\x1b[K", .{});
12
13 try T.init();
14 defer T.deinit();
15
16 try T.init_thread(null);
17 defer T.deinit_thread();
18
19 var gpa: std.heap.GeneralPurposeAllocator(.{
20 .stack_trace_frames = 100,
21 }) = .{};
22 defer _ = gpa.deinit();
23 const alloc = gpa.allocator();
24
25 var maybe_file_path: ?[]const u8 = null;
26 defer if (maybe_file_path) |path| alloc.free(path);
27
28 var cwd_buf: [std.fs.max_path_bytes]u8 = undefined;
29 const cwd = try std.posix.getcwd(&cwd_buf);
30
31 var exe_name: []const u8 = undefined;
32
33 {
34 var args = try std.process.argsWithAllocator(alloc);
35 defer args.deinit();
36
37 exe_name = try alloc.dupe(u8, std.fs.path.basename(args.next() orelse "cli"));
38 errdefer alloc.free(exe_name);
39
40 while (args.next()) |arg| {
41 maybe_file_path = try std.fs.path.resolve(alloc, &.{ cwd, arg });
42 break;
43 }
44 }
45 defer alloc.free(exe_name);
46
47 const file_path = maybe_file_path orelse {
48 std.debug.print("Usage: {s} <input file>\n", .{exe_name});
49 std.process.exit(1);
50 };
51
52 var comp: Comp = .{ .gpa = T.tracingAllocator(&alloc) };
53 try comp.init();
54 defer comp.deinit();
55 const main_file = try comp.getFile(file_path);
56
57 const teg = try main_file.getTeg(comp.gpa);
58 {
59 const stderr = std.io.getStdErr().writer();
60 try teg.dumpToWriter(stderr);
61 }
62
63 try comp.semaEntryPoint(main_file);
64
65 const output = std.io.getStdOut().writer();
66 var bufw = std.io.bufferedWriter(output);
67 const writer = bufw.writer();
68 try comp.link(writer);
69 try bufw.flush();
70
71 // var error_pos: u32 = 0;
72 // var tokens = Tokenizer.tokenize(source, alloc.allocator(), &error_pos) catch |err| {
73 // var line: usize = 1;
74 // var byte: usize = 0;
75 // var line_data: []const u8 = undefined;
76 // while (true) {
77 // const b = std.mem.indexOfScalarPos(u8, source, byte, '\n') orelse {
78 // line_data = source[byte..];
79 // break;
80 // };
81 // if (b > error_pos) {
82 // line_data = source[byte..b];
83 // break;
84 // }
85 // byte = b + 1;
86 // line += 1;
87 // }
88 // var col: usize = 1;
89 // var arrow_pos: usize = 0;
90 // while (byte < error_pos) {
91 // const b = source[byte];
92 // std.debug.assert(b != '\n');
93 // byte += std.unicode.utf8ByteSequenceLength(b) catch break;
94 // col += 1;
95
96 // arrow_pos += 1; // TODO: grapheme cluster
97 // }
98
99 // const stderr = std.io.getStdErr();
100 // const config = std.io.tty.detectConfig(stderr);
101 // var buf = std.io.bufferedWriter(stderr.writer());
102 // const writer = buf.writer();
103
104 // try config.setColor(writer, .red);
105 // try writer.writeAll("error");
106 // try config.setColor(writer, .bright_black);
107 // try writer.writeAll(": ");
108 // try config.setColor(writer, .reset);
109 // switch (err) {
110 // error.OutOfMemory => try writer.writeAll("Out of Memory"),
111 // error.InvalidUtf8 => try writer.writeAll("Invalid UTF-8"),
112 // error.InvalidSyntax => try writer.writeAll("Syntax Error"),
113 // error.TrailingNumericSeparator => try writer.writeAll("Trailing Numeric Separator"),
114 // error.UnterminatedString => try writer.writeAll("Unterminated String"),
115 // error.InvalidEscapeSequence => try writer.writeAll("Invalid Escape Sequence"),
116 // error.InvalidEscapeCodepoint => try writer.writeAll("Invalid Escape Codepoint"),
117 // error.GitConflict => try writer.writeAll("Found Git Conflict"),
118 // }
119 // try writer.print(" at {d}:{d}\n ", .{ line, col });
120 // try writer.writeAll(line_data);
121 // try writer.writeAll("\n ");
122 // try writer.writeByteNTimes(' ', col);
123
124 // try config.setColor(writer, .bright_green);
125 // try writer.writeAll("^");
126 // try config.setColor(writer, .reset);
127 // try writer.writeAll("\n");
128
129 // try buf.flush();
130 // return err;
131 // };
132 // defer tokens.deinit(alloc.allocator());
133
134 // // std.debug.print("tokens.len = {d}\n", .{tokens.len});
135 // // std.debug.print("token list bytes = {d}\n", .{tokens.toMultiArrayList().allocatedBytes().len});
136 // // std.debug.print("source bytes = {d}\n", .{source.len});
137 // // for (
138 // // tokens.items(.tag),
139 // // tokens.items(.offset),
140 // // ) |tag, offset| {
141 // // std.debug.print(".tag = {s} .offset = {d}\n", .{
142 // // @tagName(tag),
143 // // offset,
144 // // });
145 // // }
146
147 // const ast = ast: {
148 // var p = Parse.init(alloc.allocator(), source, tokens);
149 // errdefer p.deinit();
150 // try p.nodes.ensureUnusedCapacity(alloc.allocator(), 100);
151 // p.parseRoot() catch |err| {
152 // const tok = p.fullToken(p.pos);
153
154 // std.debug.print("ast_error = {?}\n", .{p.ast_error});
155
156 // std.debug.print("tok: {s} - '{s}' - {d}\n", .{
157 // @tagName(tok.tag),
158 // tok.slice(p.source),
159 // tok.offset,
160 // });
161
162 // return err;
163 // };
164 // break :ast try p.finalize(tokens);
165 // };
166
167 // try ast.dumpToWriter(std.io.getStdOut().writer());
168
169 // const teg = try Transform.transform(ast, alloc.allocator());
170 // std.debug.print("teg.insts.len = {d}\n", .{teg.insts.len});
171 // std.debug.print("teg.string_bytes.len = {d}\n", .{teg.string_bytes.len});
172 // std.debug.print("teg.extra.len = {d}\n", .{teg.extra.len});
173
174 // for (teg.insts.items(.tag), teg.insts.items(.data)) |tag, data| {
175 // std.debug.print("{s} - {}\n", .{
176 // @tagName(tag),
177 // data,
178 // });
179 // }
180}
src/compiler/unicode_props.zig created+103
......@@ -0,0 +1,103 @@
1//! Wrapper around the 'zg' PropsData library.
2const std = @import("std");
3const PropsData = @import("zg:PropsData");
4const T = @import("tracer");
5
6var props: ?PropsData = undefined;
7var props_memory: []u8 = undefined;
8
9const raw_allocator = if (@import("builtin").target.isWasm())
10 std.heap.wasm_allocator
11else
12 std.heap.page_allocator;
13
14pub fn init() !void {
15 if (props != null) return;
16
17 const trace = T.trace(@src());
18 defer trace.end();
19
20 // We know exactly how much this uses, so let's preallocate these pages and use a FBA.
21 const allocation_size = 8704 + 34560 + 8704 + 17664 + 8704 + 25856;
22
23 if (allocation_size == 0) {
24 var logging_alloc = std.heap.loggingAllocator(std.heap.page_allocator);
25 props = try PropsData.init(logging_alloc.allocator());
26 std.log.info("please write the above numbers into unicode.zig to prealloc this", .{});
27 std.process.exit(1);
28 }
29
30 props_memory = try raw_allocator.alloc(u8, allocation_size);
31 errdefer std.heap.page_allocator.free(props_memory);
32
33 var fba = std.heap.FixedBufferAllocator.init(props_memory);
34 props = try PropsData.init(fba.allocator());
35 std.debug.assert(fba.end_index == allocation_size);
36}
37
38pub fn deinit() void {
39 std.debug.assert(props != null);
40 raw_allocator.free(props_memory);
41 props = null;
42 props_memory = undefined;
43}
44
45/// True if `cp` is a mathematical symbol.
46pub inline fn isMath(cp: u21) bool {
47 return props.?.isMath(cp);
48}
49
50/// True if `cp` is an alphabetic character.
51pub inline fn isAlphabetic(cp: u21) bool {
52 return props.?.isAlphabetic(cp);
53}
54
55/// True if `cp` is a valid identifier start character.
56pub inline fn isIdStart(cp: u21) bool {
57 return props.?.isIdStart(cp);
58}
59
60/// True if `cp` is a valid identifier continuation character.
61pub inline fn isIdContinue(cp: u21) bool {
62 return props.?.isIdContinue(cp);
63}
64
65/// True if `cp` is a valid extended identifier start character.
66pub inline fn isXidStart(cp: u21) bool {
67 return props.?.isXidStart(cp);
68}
69
70/// True if `cp` is a valid extended identifier continuation character.
71pub inline fn isXidContinue(cp: u21) bool {
72 return props.?.isXidContinue(cp);
73}
74
75/// True if `cp` is a whitespace character.
76pub inline fn isWhitespace(cp: u21) bool {
77 return props.?.isWhitespace(cp);
78}
79
80/// True if `cp` is a hexadecimal digit.
81pub inline fn isHexDigit(cp: u21) bool {
82 return props.?.isHexDigit(cp);
83}
84
85/// True if `cp` is a diacritic mark.
86pub inline fn isDiacritic(cp: u21) bool {
87 return props.?.isDiacritic(cp);
88}
89
90/// True if `cp` is numeric.
91pub inline fn isNumeric(cp: u21) bool {
92 return props.?.isNumeric(cp);
93}
94
95/// True if `cp` is a digit.
96pub inline fn isDigit(cp: u21) bool {
97 return props.?.isDigit(cp);
98}
99
100/// True if `cp` is decimal.
101pub inline fn isDecimal(cp: u21) bool {
102 return props.?.isDecimal(cp);
103}
src/compiler/util.zig created+20
......@@ -0,0 +1,20 @@
1/// TODO: move to io utility file
2pub fn detectTTYFromWriter(writer: anytype) std.io.tty.Config {
3 if (@hasField(@TypeOf(writer), "unbuffered_writer")) {
4 return detectTTYFromWriter(writer.unbuffered_writer);
5 }
6 if (@TypeOf(writer) == std.fs.File.Writer) {
7 return std.io.tty.detectConfig(writer.context);
8 }
9 if (@TypeOf(writer) == std.io.AnyWriter) {
10 const writeFn = comptime (std.fs.File{ .handle = undefined })
11 .writer().any().writeFn;
12 if (writer.writeFn == writeFn) {
13 const file: *const std.fs.File = @alignCast(@ptrCast(writer.context));
14 return std.io.tty.detectConfig(file.*);
15 }
16 }
17 return .no_color;
18}
19
20const std = @import("std");
src/wasm/enum_codegen.zig created+28
......@@ -0,0 +1,28 @@
1const std = @import("std");
2const Tokenizer = @import("Tokenizer.zig");
3const Ast = @import("Ast.zig");
4const Teg = @import("Teg.zig");
5// const InternPool = @import("InternPool.zig");
6
7pub fn main() !void {
8 var buf = std.io.bufferedWriter(std.io.getStdOut().writer());
9 defer buf.flush() catch {};
10 const w = buf.writer();
11
12 inline for (.{
13 .{ "TokenTag", Tokenizer.Token.Tag },
14 .{ "NodeTag", Ast.Node.Tag },
15 .{ "TegInstTag", Teg.Inst.Tag },
16 .{ "TegInstRef", Teg.Inst.Ref },
17 // .{ "InternPoolIndex", InternPool.Index },
18 }) |decl| {
19 const name, const enum_type = decl;
20
21 try w.print("export const {s} = {{\n", .{name});
22 for (std.enums.values(enum_type)) |tag| {
23 try w.print(" {d}: \"{s}\",\n", .{ @intFromEnum(tag), @tagName(tag) });
24 }
25 try w.writeAll("};\n");
26 try w.writeAll("\n");
27 }
28}
src/wasm/wasm_api.zig created+32
......@@ -0,0 +1,32 @@
1//! A subset of the compiler is exposed to wasm
2//! This powers the playground
3
4extern fn iterateTokenCallback(
5 tag: i32,
6 offset: ByteOffset,
7) void;
8
9export fn iterateTokens(
10 source_bytes: [*]const u8,
11 source_len: u32,
12) callconv(.C) i32 {
13 unicode_props.init() catch @panic("unicode_props.init failed");
14
15 var tokenizer: Tokenizer = .{ .code = source_bytes[0..source_len], .pos = 0 };
16 tokenizer.skipWhitespace();
17 while (tokenizer.next() catch {
18 return @intCast(tokenizer.pos);
19 }) |token| {
20 iterateTokenCallback(@intFromEnum(token.tag), token.offset);
21 tokenizer.skipWhitespace();
22 }
23
24 return -1;
25}
26
27const std = @import("std");
28const wasm_allocator = std.heap.wasm_allocator;
29const Tokenizer = @import("Tokenizer.zig");
30const Token = Tokenizer.Token;
31const ByteOffset = Tokenizer.ByteOffset;
32const unicode_props = @import("unicode_props.zig");