diff --git a/.github/logo.png b/.github/logo.png
deleted file mode 100644
index 46e7e472cc5655f578d3cd2e873d230107999375..0000000000000000000000000000000000000000
Binary files a/.github/logo.png and /dev/null differ
diff --git a/.gitignore b/.gitignore
index 4143657114f43375f7bb260327bd203355dcd140..6c08354510e655ecc6f55e468a65aba4c006d11e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,10 @@
-target
-*.js
+# If you have OS-specific or editor-specific files to ignore,
+# such as *.swp or .DS_Store, put those in your global
+# ~/.gitignore and put this in your ~/.gitconfig:
+#
+# [core]
+# excludesfile = ~/.gitignore
+#
+
+zig-out
+zig-cache
diff --git a/Cargo.lock b/Cargo.lock
deleted file mode 100644
index bc884459d74ada74428241bd211efc2c04a4dda8..0000000000000000000000000000000000000000
--- a/Cargo.lock
+++ /dev/null
@@ -1,759 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 3
-
-[[package]]
-name = "addr2line"
-version = "0.21.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
-dependencies = [
- "gimli",
-]
-
-[[package]]
-name = "adler"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
-
-[[package]]
-name = "anstream"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6"
-dependencies = [
- "anstyle",
- "anstyle-parse",
- "anstyle-query",
- "anstyle-wincon",
- "colorchoice",
- "utf8parse",
-]
-
-[[package]]
-name = "anstyle"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"
-
-[[package]]
-name = "anstyle-parse"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c"
-dependencies = [
- "utf8parse",
-]
-
-[[package]]
-name = "anstyle-query"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648"
-dependencies = [
- "windows-sys",
-]
-
-[[package]]
-name = "anstyle-wincon"
-version = "3.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7"
-dependencies = [
- "anstyle",
- "windows-sys",
-]
-
-[[package]]
-name = "ast"
-version = "0.1.0"
-dependencies = [
- "log",
- "miette",
- "num",
- "thiserror",
- "tokenizer",
-]
-
-[[package]]
-name = "autocfg"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
-
-[[package]]
-name = "backtrace"
-version = "0.3.69"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837"
-dependencies = [
- "addr2line",
- "cc",
- "cfg-if",
- "libc",
- "miniz_oxide",
- "object",
- "rustc-demangle",
-]
-
-[[package]]
-name = "backtrace-ext"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50"
-dependencies = [
- "backtrace",
-]
-
-[[package]]
-name = "bitflags"
-version = "2.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
-
-[[package]]
-name = "cc"
-version = "1.0.83"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "cfg-if"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
-
-[[package]]
-name = "clap"
-version = "4.4.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33e92c5c1a78c62968ec57dbc2440366a2d6e5a23faf829970ff1585dc6b18e2"
-dependencies = [
- "clap_builder",
- "clap_derive",
-]
-
-[[package]]
-name = "clap_builder"
-version = "4.4.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4323769dc8a61e2c39ad7dc26f6f2800524691a44d74fe3d1071a5c24db6370"
-dependencies = [
- "anstream",
- "anstyle",
- "clap_lex",
- "strsim",
-]
-
-[[package]]
-name = "clap_derive"
-version = "4.4.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442"
-dependencies = [
- "heck",
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "clap_lex"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
-
-[[package]]
-name = "cli"
-version = "0.1.0"
-dependencies = [
- "ast",
- "clap",
- "js_ast",
- "js_printer",
- "log",
- "miette",
- "parser",
- "sema",
- "thiserror",
- "tokenizer",
-]
-
-[[package]]
-name = "colorchoice"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
-
-[[package]]
-name = "errno"
-version = "0.3.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245"
-dependencies = [
- "libc",
- "windows-sys",
-]
-
-[[package]]
-name = "form_urlencoded"
-version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
-dependencies = [
- "percent-encoding",
-]
-
-[[package]]
-name = "gimli"
-version = "0.28.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
-
-[[package]]
-name = "heck"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
-
-[[package]]
-name = "hermit-abi"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
-
-[[package]]
-name = "idna"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
-dependencies = [
- "unicode-bidi",
- "unicode-normalization",
-]
-
-[[package]]
-name = "is-terminal"
-version = "0.4.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455"
-dependencies = [
- "hermit-abi",
- "rustix",
- "windows-sys",
-]
-
-[[package]]
-name = "is_ci"
-version = "1.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "616cde7c720bb2bb5824a224687d8f77bfd38922027f01d825cd7453be5099fb"
-
-[[package]]
-name = "js_ast"
-version = "0.1.0"
-dependencies = [
- "ast",
- "log",
- "miette",
- "num",
- "thiserror",
-]
-
-[[package]]
-name = "js_printer"
-version = "0.1.0"
-dependencies = [
- "js_ast",
- "miette",
- "num",
- "thiserror",
-]
-
-[[package]]
-name = "libc"
-version = "0.2.152"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7"
-
-[[package]]
-name = "linux-raw-sys"
-version = "0.4.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456"
-
-[[package]]
-name = "log"
-version = "0.0.0"
-dependencies = [
- "miette",
- "thiserror",
- "url",
-]
-
-[[package]]
-name = "memchr"
-version = "2.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
-
-[[package]]
-name = "miette"
-version = "5.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e"
-dependencies = [
- "backtrace",
- "backtrace-ext",
- "is-terminal",
- "miette-derive",
- "once_cell",
- "owo-colors",
- "supports-color",
- "supports-hyperlinks",
- "supports-unicode",
- "terminal_size",
- "textwrap",
- "thiserror",
- "unicode-width",
-]
-
-[[package]]
-name = "miette-derive"
-version = "5.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "miniz_oxide"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
-dependencies = [
- "adler",
-]
-
-[[package]]
-name = "num"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af"
-dependencies = [
- "num-bigint",
- "num-complex",
- "num-integer",
- "num-iter",
- "num-rational",
- "num-traits",
-]
-
-[[package]]
-name = "num-bigint"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0"
-dependencies = [
- "autocfg",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "num-complex"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214"
-dependencies = [
- "num-traits",
-]
-
-[[package]]
-name = "num-integer"
-version = "0.1.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
-dependencies = [
- "autocfg",
- "num-traits",
-]
-
-[[package]]
-name = "num-iter"
-version = "0.1.43"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"
-dependencies = [
- "autocfg",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "num-rational"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0"
-dependencies = [
- "autocfg",
- "num-bigint",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "num-traits"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
-dependencies = [
- "autocfg",
-]
-
-[[package]]
-name = "object"
-version = "0.32.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "once_cell"
-version = "1.19.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
-
-[[package]]
-name = "owo-colors"
-version = "3.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
-
-[[package]]
-name = "parser"
-version = "0.1.0"
-dependencies = [
- "ast",
- "log",
- "miette",
- "num",
- "thiserror",
- "tokenizer",
-]
-
-[[package]]
-name = "percent-encoding"
-version = "2.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.76"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
-dependencies = [
- "proc-macro2",
-]
-
-[[package]]
-name = "rustc-demangle"
-version = "0.1.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
-
-[[package]]
-name = "rustix"
-version = "0.38.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316"
-dependencies = [
- "bitflags",
- "errno",
- "libc",
- "linux-raw-sys",
- "windows-sys",
-]
-
-[[package]]
-name = "sema"
-version = "0.1.0"
-dependencies = [
- "ast",
- "miette",
- "num",
- "thiserror",
-]
-
-[[package]]
-name = "smawk"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c"
-
-[[package]]
-name = "strsim"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
-
-[[package]]
-name = "supports-color"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d6398cde53adc3c4557306a96ce67b302968513830a77a95b2b17305d9719a89"
-dependencies = [
- "is-terminal",
- "is_ci",
-]
-
-[[package]]
-name = "supports-hyperlinks"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f84231692eb0d4d41e4cdd0cabfdd2e6cd9e255e65f80c9aa7c98dd502b4233d"
-dependencies = [
- "is-terminal",
-]
-
-[[package]]
-name = "supports-unicode"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b6c2cb240ab5dd21ed4906895ee23fe5a48acdbd15a3ce388e7b62a9b66baf7"
-dependencies = [
- "is-terminal",
-]
-
-[[package]]
-name = "syn"
-version = "2.0.48"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "terminal_size"
-version = "0.1.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df"
-dependencies = [
- "libc",
- "winapi",
-]
-
-[[package]]
-name = "textwrap"
-version = "0.15.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7b3e525a49ec206798b40326a44121291b530c963cfb01018f63e135bac543d"
-dependencies = [
- "smawk",
- "unicode-linebreak",
- "unicode-width",
-]
-
-[[package]]
-name = "thiserror"
-version = "1.0.56"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad"
-dependencies = [
- "thiserror-impl",
-]
-
-[[package]]
-name = "thiserror-impl"
-version = "1.0.56"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "tinyvec"
-version = "1.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
-dependencies = [
- "tinyvec_macros",
-]
-
-[[package]]
-name = "tinyvec_macros"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
-
-[[package]]
-name = "tokenizer"
-version = "0.1.0"
-dependencies = [
- "log",
- "miette",
- "thiserror",
-]
-
-[[package]]
-name = "unicode-bidi"
-version = "0.3.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
-
-[[package]]
-name = "unicode-ident"
-version = "1.0.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
-
-[[package]]
-name = "unicode-linebreak"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f"
-
-[[package]]
-name = "unicode-normalization"
-version = "0.1.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921"
-dependencies = [
- "tinyvec",
-]
-
-[[package]]
-name = "unicode-width"
-version = "0.1.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85"
-
-[[package]]
-name = "url"
-version = "2.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633"
-dependencies = [
- "form_urlencoded",
- "idna",
- "percent-encoding",
-]
-
-[[package]]
-name = "utf8parse"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
-
-[[package]]
-name = "winapi"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
-dependencies = [
- "winapi-i686-pc-windows-gnu",
- "winapi-x86_64-pc-windows-gnu",
-]
-
-[[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-
-[[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-
-[[package]]
-name = "windows-sys"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
-dependencies = [
- "windows-targets",
-]
-
-[[package]]
-name = "windows-targets"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd"
-dependencies = [
- "windows_aarch64_gnullvm",
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_gnullvm",
- "windows_x86_64_msvc",
-]
-
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"
-
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"
-
-[[package]]
-name = "windows_i686_gnu"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"
-
-[[package]]
-name = "windows_i686_msvc"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"
-
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"
-
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"
-
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
diff --git a/Cargo.toml b/Cargo.toml
deleted file mode 100644
index 51b88c742a378171aa0093edefae5b956af2eb22..0000000000000000000000000000000000000000
--- a/Cargo.toml
+++ /dev/null
@@ -1,26 +0,0 @@
-[workspace.package]
-version = "0.1.0"
-edition = "2021"
-license = "MIT"
-authors = ["Lino Le Van", "Dave Caruso"]
-repository = "https://github.com/lino-levan/blight"
-
-[workspace]
-resolver = "2"
-members = [
- "crates/ast",
- "crates/cli",
- "crates/js_ast",
- "crates/js_printer",
- "crates/log",
- "crates/parser",
- "crates/sema",
- "crates/tokenizer",
-]
-
-[workspace.dependencies]
-clap = { version = "4.4.12", features = ["derive"] }
-miette = { version = "5.10.0", features = ["fancy"] }
-thiserror = "1.0.53"
-num = "0.4.1"
-url = "2.2.2"
diff --git a/README.md b/README.md
index 25323d8ec84c7b31e32fb7e35de3bc157de8d9de..753fb38ee29536347d8cc9a7a7bea9b702706c0e 100644
--- a/README.md
+++ b/README.md
@@ -1,19 +1,79 @@
-
+# Compiler
-# Blight
+This language is a mix of features from JavaScript, Rust, and Zig. Some notable features carried over from each include:
-Blight sure is a programming language.
+- **Rust** - Expressive control flow that is easy to write.
+- **Zig** - Execute any code at compile time, everything is an expression.
+- **JavaScript** - Garbage collected values with a versitile type system.
-## Developing
+## Standard Library
-Compiling a file:
+Currently, there is no standard library. One will be added later.
-```bash
-cargo run compile --stdout ./examples/basics/fib.blight
+## Hello World
+
+```
+let print = @js("console").log;
+print("Hello, world!");
+```
+
+The `print` function is implemented by getting an external reference to the `console` JavaScript global, and then retriving the `log` property from it.
+
+## Primative Data Types
+
+| name | ts equivilent | description |
+| ---------- | --------------------- | ---------------------------------------------- |
+| `int` | `number` | a number in signed 32-bit range |
+| `float` | `number` | a 64-bit floating point number |
+| `bool` | `boolean` | either `true` or `false` |
+| `string` | `string` | a sequence of text characters |
+| `void` | `undefined` or `null` | detectable lack of a value |
+| `noreturn` | `never` | control flow prevents this value from existing |
+
+> **note**: `bigint` is planned but not added
+
+## Variables
+
+There are two types of variables, `let` and `var`.
+
+`let` bindings cannot be overridden, and should be thought of as aliases of the expression within. Referring to these identifiers is equivilent to inlining it's where it was used, aside from side effect duplication.
+
+```
+let hello: int = 1;
+let world = hello + 2;
+let hello = hello * world;
+
+@js("console").log(hello);
```
-Running the compiler on a file:
+`var` bindings create mutable memory locations at runtime. This means they can be re-assigned to in arbitrary ways.
-```bash
-cargo run run ./examples/basics/fib.blight
+```
+var message = "first message";
+
+if @js("Math").random() > 0.5 {
+ message = "second message"
+}
+
+@js("console").log(message);
+```
+
+`let` is preferred whenever possible, as it will lead to more readable code, and the compiler is able to optimize it better. The above can be rewritten as:
+
+```
+let message = if @js("Math").random() > 0.5 {
+ "second message"
+} else {
+ "first message"
+};
+
+@js("console").log(message);
+```
+
+The initializer for a `let` and `var` are always required, but the type annotations can be omitted when the data type can be inferred. For `let`, this is always the case. `var`, being runtime-mutable memory, only supports runtime representable data types. This means for numerical literals, an explicit type annotation is needed to determine weather to store an integer or a floating point number.
+
+```
+var a = 5; // error
+var b: int = 5; // ok
+var c: float = 5; // ok
```
diff --git a/build.zig b/build.zig
new file mode 100644
index 0000000000000000000000000000000000000000..4901620fb6c1ec93c4316653b2689d44456b7adc
--- /dev/null
+++ b/build.zig
@@ -0,0 +1,96 @@
+const std = @import("std");
+
+pub fn build(b: *std.Build) void {
+ b.reference_trace = 20;
+
+ const target = b.standardTargetOptions(.{});
+ const optimize = b.standardOptimizeOption(.{});
+
+ const dep_opts = .{ .target = target, .optimize = optimize };
+
+ const zg = b.dependency("zg", dep_opts);
+ const tracer = b.dependency("tracer", dep_opts);
+ // const ada = b.dependency("ada", dep_opts);
+
+ const exe = b.addExecutable(.{
+ .name = "compiler",
+ .root_source_file = b.path("src/compiler/main.zig"),
+ .target = target,
+ .optimize = optimize,
+ });
+ b.installArtifact(exe);
+
+ exe.root_module.addImport("zg:PropsData", zg.module("PropsData"));
+ exe.root_module.addImport("tracer", tracer.module("tracer"));
+ // if (!target.result.isWasm()) {
+ // exe.root_module.addImport("ada", ada.module("ada"));
+ // }
+
+ {
+ const run_cmd = b.addRunArtifact(exe);
+ run_cmd.step.dependOn(b.getInstallStep());
+ if (b.args) |args| {
+ run_cmd.addArgs(args);
+ }
+ const run_step = b.step("compiler", "Run compiler CLI");
+ run_step.dependOn(&run_cmd.step);
+ }
+
+ // const wasm_api = b.addExecutable(.{
+ // .name = "compiler",
+ // .root_source_file = b.path("src/wasm_api.zig"),
+ // .target = b.resolveTargetQuery(.{
+ // .cpu_arch = .wasm32,
+ // .os_tag = .freestanding,
+ // }),
+ // .optimize = optimize,
+ // });
+ // wasm_api.root_module.export_symbol_names = &[_][]const u8{
+ // "iterateTokens",
+ // };
+ // wasm_api.entry = .disabled;
+ // const wasm_step = b.step("wasm", "Build wasm api");
+ // b.installArtifact(wasm_api);
+ // wasm_step.dependOn(&wasm_api.step);
+ // wasm_api.root_module.addImport("zg:PropsData", zg.module("PropsData"));
+
+ const release = b.step("release", "Package releases");
+ inline for (.{
+ b.resolveTargetQuery(.{ .cpu_arch = .x86_64, .os_tag = .linux, .abi = .musl }),
+ b.resolveTargetQuery(.{ .cpu_arch = .aarch64, .os_tag = .linux, .abi = .musl }),
+ b.resolveTargetQuery(.{ .cpu_arch = .x86_64, .os_tag = .macos }),
+ b.resolveTargetQuery(.{ .cpu_arch = .aarch64, .os_tag = .macos }),
+ b.resolveTargetQuery(.{ .cpu_arch = .x86_64, .os_tag = .windows }),
+ b.resolveTargetQuery(.{ .cpu_arch = .aarch64, .os_tag = .windows }),
+ }) |cross_target| {
+ const name = std.fmt.allocPrint(
+ b.allocator,
+ "{s}-{s}",
+ .{
+ @tagName(cross_target.result.cpu.arch),
+ @tagName(cross_target.result.os.tag),
+ },
+ ) catch @panic("OOM");
+
+ const cross_exe = b.addExecutable(.{
+ .name = "compiler",
+ .root_source_file = b.path("src/compiler/main.zig"),
+ .target = cross_target,
+ .optimize = .ReleaseSafe,
+ .strip = true,
+ });
+ cross_exe.root_module.addImport("zg:PropsData", zg.module("PropsData"));
+ cross_exe.root_module.addImport("tracer", tracer.module("tracer"));
+
+ release.dependOn(&b.addInstallFile(
+ cross_exe.getEmittedBin(),
+ b.pathJoin(&.{
+ name,
+ std.mem.concat(b.allocator, u8, &.{
+ "compiler",
+ cross_target.result.exeFileExt(),
+ }) catch @panic("OOM"),
+ }),
+ ).step);
+ }
+}
diff --git a/build.zig.zon b/build.zig.zon
new file mode 100644
index 0000000000000000000000000000000000000000..3c75338bb1c8bba9e1d92569fc1c914bd19e7d8e
--- /dev/null
+++ b/build.zig.zon
@@ -0,0 +1,22 @@
+.{
+ .name = "",
+ .version = "0.0.0",
+
+ .minimum_zig_version = "0.12.0",
+
+ .dependencies = .{
+ .zg = .{
+ .url = "https://codeberg.org/dude_the_builder/zg/archive/16735685fcc3410de361ba3411788ad1fb4fe188.tar.gz",
+ .hash = "1220fe9ac5cdb41833d327a78745614e67d472469f8666567bd8cf9f5847a52b1c51",
+ },
+ .tracer = .{
+ .url = "https://github.com/paperdave/zig-tracer/archive/d4c2eff040e9e69b9f08d2175b315f3055ad7ba3.tar.gz",
+ .hash = "12202d60766a46ceb4074ce5651a58a11f2f6b32927476a55c6889805073e8e3d261",
+ },
+ },
+ .paths = .{
+ "build.zig",
+ "build.zig.zon",
+ "src",
+ },
+}
diff --git a/crates/ast/Cargo.toml b/crates/ast/Cargo.toml
deleted file mode 100644
index fe49e948853cee5c92a9a0ca399f13ed8b2ec4ed..0000000000000000000000000000000000000000
--- a/crates/ast/Cargo.toml
+++ /dev/null
@@ -1,15 +0,0 @@
-[package]
-name = "ast"
-version = "0.1.0"
-edition = "2021"
-
-[lib]
-path = "./lib.rs"
-
-[dependencies]
-miette.workspace = true
-thiserror.workspace = true
-num.workspace = true
-
-tokenizer = { path = "../tokenizer" } # Do not depend on the tokenizer
-log = { path = "../log" }
diff --git a/crates/ast/ast.rs b/crates/ast/ast.rs
deleted file mode 100644
index 6a30596e334101be04882142df1432fa7ff1795c..0000000000000000000000000000000000000000
--- a/crates/ast/ast.rs
+++ /dev/null
@@ -1,211 +0,0 @@
-use log::Range;
-
-use crate::types::{Literal, Type};
-use crate::{BinaryOperator, UnaryOperator};
-
-#[derive(Debug, Clone)]
-pub enum BuiltinId {
- Raw,
-}
-
-#[derive(Debug, Clone)]
-pub struct Module {
- pub comment: Option,
- pub statements: Vec,
-}
-
-#[derive(Debug, Clone)]
-pub struct Block {
- pub statements: Vec,
- pub has_implicit_return: bool,
-}
-
-#[derive(Debug, Clone)]
-pub struct Param {
- pub name: String,
- pub param_type: Type,
-}
-
-#[derive(Debug, Clone)]
-
-pub struct VarDecl {
- pub name: String,
- pub data_type: Type,
- pub init: Expr,
- pub is_constant: bool,
- pub is_pub: bool,
- pub range: Range,
- pub comment: Option,
-}
-
-#[derive(Debug, Clone)]
-pub struct FnDecl {
- pub name: String,
- pub return_type: Type,
- pub params: Vec,
- pub body: Expr,
- pub is_pub: bool,
- pub range: Range,
- pub comment: Option,
-}
-
-#[derive(Debug, Clone)]
-pub struct Expr {
- pub data: ExprData,
- pub range: Range,
- pub comment: Option,
-}
-
-impl Expr {
- pub fn new(data: ExprData, range: Range, comment: Option) -> Self {
- Self {
- data,
- range,
- comment,
- }
- }
-}
-
-#[derive(Debug, Clone)]
-pub struct DocComment {
- pub data: String,
- pub range: Range,
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct Comment {
- pub data: String,
-}
-
-impl Comment {
- pub fn format(&self) -> String {
- let mut lines = self.data.lines();
- let first_line = lines.next().unwrap();
- let mut result = format!("// {}", first_line);
- for line in lines {
- result.push_str(&format!("\n// {}", line));
- }
- result
- }
-
- pub fn to_doc_comment(self, range: Range) -> DocComment {
- DocComment {
- data: self.data,
- range,
- }
- }
-}
-
-impl Into for DocComment {
- fn into(self) -> Comment {
- Comment { data: self.data }
- }
-}
-
-impl DocComment {
- pub fn format(&self) -> String {
- let mut lines = self.data.lines();
- let first_line = lines.next().unwrap();
- let mut result = format!("/// {}", first_line);
- for line in lines {
- result.push_str(&format!("\n/// {}", line));
- }
- result
- }
-}
-
-#[derive(Debug, Clone)]
-pub struct BlockWithRange {
- pub block: Block,
- pub range: Range,
-}
-
-#[derive(Debug, Clone)]
-pub struct If {
- pub condition: Box,
- pub yes: BlockWithRange,
- pub no: Option,
-}
-
-#[derive(Debug, Clone)]
-pub enum BlockOrIf {
- Block(BlockWithRange),
- If(Box),
-}
-
-#[derive(Debug, Clone)]
-pub enum ExprData {
- Block(Block),
- Binary {
- op: BinaryOperator,
- left: Box,
- right: Box,
- },
- Unary {
- op: UnaryOperator,
- value: Box,
- },
- Fn {
- name: String,
- params: Vec,
- return_type: Type,
- body: Box,
- },
- Call {
- base: Box,
- arguments: Vec,
- },
- Field {
- base: Box,
- field: Box,
- },
- Builtin {
- id: BuiltinId,
- arguments: Vec,
- },
- If(If),
- Return(Option>),
- Literal(Literal),
- Identifier(String),
-}
-
-#[derive(Debug, Clone)]
-pub enum Statement {
- Expr(Expr),
- VarDecl(VarDecl),
- FnDecl(FnDecl),
- Comment(Comment),
-}
-
-impl BuiltinId {
- pub fn display_name(&self) -> &'static str {
- match self {
- Self::Raw => "@raw",
- }
- }
-
- pub fn from_str(name: &str) -> Option {
- match name {
- "@raw" => Some(Self::Raw),
- _ => None,
- }
- }
-}
-
-impl Statement {
- /// This is stupid function im sorry
- pub fn require_semicolon(&self, semicolon_hint: bool) -> bool {
- match &self {
- Statement::Expr(_) => !semicolon_hint,
- Statement::VarDecl(_) => true,
- _ => false,
- }
- }
-
- pub fn is_expression(&self) -> bool {
- match &self {
- Statement::Expr(_) => true,
- _ => false,
- }
- }
-}
diff --git a/crates/ast/lib.rs b/crates/ast/lib.rs
deleted file mode 100644
index c35262a7f11a1125135720a5efee113eb1ec8c29..0000000000000000000000000000000000000000
--- a/crates/ast/lib.rs
+++ /dev/null
@@ -1,7 +0,0 @@
-mod ast;
-mod operators;
-mod types;
-
-pub use ast::*;
-pub use operators::*;
-pub use types::*;
diff --git a/crates/ast/operators.rs b/crates/ast/operators.rs
deleted file mode 100644
index 27cf0b1a837d2f1f40b45af57026cc4700ed50e2..0000000000000000000000000000000000000000
--- a/crates/ast/operators.rs
+++ /dev/null
@@ -1,316 +0,0 @@
-use tokenizer::TokenType;
-
-use crate::Literal;
-
-#[derive(Debug, Clone, PartialEq)]
-pub enum BinaryOperator {
- Add,
- Sub,
- Mul,
- Div,
- Pow,
- Eq,
- Gt,
- Lt,
- Gte,
- Lte,
- Lshift,
- Rshift,
- RshiftUnsigned,
- BitAnd,
- BitOr,
- BitXor,
- Mod,
- NotEq,
- And,
- Or,
-
- Assign,
- AssignAdd,
- AssignSub,
- AssignMul,
- AssignDiv,
- AssignPow,
- AssignLshift,
- AssignRshift,
- AssignRshiftUnsigned,
- AssignBitAnd,
- AssignBitOr,
- AssignBitXor,
- AssignMod,
- AssignAnd,
- AssignOr,
-}
-
-#[derive(Debug, Clone, PartialEq)]
-pub enum UnaryOperator {
- Neg,
- Not,
- BitNot,
-}
-
-impl BinaryOperator {
- pub fn to_str(&self) -> &'static str {
- match self {
- Self::Add => "+",
- Self::Sub => "-",
- Self::Div => "/",
- Self::Mul => "*",
- Self::Pow => "**",
- Self::Eq => "==",
- Self::Gt => ">",
- Self::Lt => "<",
- Self::Gte => ">=",
- Self::Lte => "<=",
- Self::Lshift => "<<",
- Self::Rshift => ">>",
- Self::RshiftUnsigned => ">>>",
- Self::BitAnd => "&",
- Self::BitOr => "|",
- Self::BitXor => "^",
- Self::Mod => "%",
- Self::NotEq => "!=",
- Self::And => "&&",
- Self::Or => "||",
- Self::AssignAdd => "+=",
- Self::AssignSub => "-=",
- Self::AssignDiv => "/=",
- Self::AssignMul => "*=",
- Self::AssignPow => "**=",
- Self::Assign => "=",
- Self::AssignLshift => "<<=",
- Self::AssignRshift => ">>=",
- Self::AssignRshiftUnsigned => ">>>=",
- Self::AssignBitAnd => "&=",
- Self::AssignBitOr => "|=",
- Self::AssignBitXor => "^=",
- Self::AssignMod => "%=",
- Self::AssignAnd => "&&=",
- Self::AssignOr => "||=",
- }
- }
-
- pub fn to_string(&self) -> String {
- String::from(self.to_str())
- }
-
- pub fn from_token(token: &TokenType) -> Option {
- match &token {
- TokenType::And => Some(Self::BitAnd),
- TokenType::AndAnd => Some(Self::And),
- TokenType::BangEqual => Some(Self::NotEq),
- TokenType::Caret => Some(Self::BitXor),
- TokenType::EqualEqual => Some(Self::Eq),
- TokenType::Gt => Some(Self::Gt),
- TokenType::GtEqual => Some(Self::Gte),
- TokenType::GtGt => Some(Self::Rshift),
- TokenType::GtGtGt => Some(Self::RshiftUnsigned),
- TokenType::Lt => Some(Self::Lt),
- TokenType::LtEqual => Some(Self::Lte),
- TokenType::LtLt => Some(Self::Lshift),
- TokenType::Minus => Some(Self::Sub),
- TokenType::Or => Some(Self::BitOr),
- TokenType::OrOr => Some(Self::Or),
- TokenType::Percent => Some(Self::Mod),
- TokenType::Plus => Some(Self::Add),
- TokenType::Slash => Some(Self::Div),
- TokenType::Star => Some(Self::Mul),
- TokenType::StarStar => Some(Self::Pow),
- _ => None,
- }
- }
-
- pub fn level(&self) -> OperatorLevel {
- OperatorLevel::from(self)
- }
-
- /// Error case of this means 'types do not allow this operation'
- pub fn perform(&self, left: Literal, right: Literal) -> Result {
- match self {
- BinaryOperator::Eq => Ok(Literal::Bool(left == right)),
- BinaryOperator::NotEq => Ok(Literal::Bool(left != right)),
-
- BinaryOperator::Or | BinaryOperator::And => match (left, right) {
- (Literal::Bool(a), Literal::Bool(b)) => Ok(Literal::Bool(match self {
- BinaryOperator::Or => a || b,
- BinaryOperator::And => a && b,
- _ => unreachable!(),
- })),
- _ => Err(()),
- },
-
- BinaryOperator::Add
- | BinaryOperator::Sub
- | BinaryOperator::Mul
- | BinaryOperator::Div
- | BinaryOperator::Pow
- | BinaryOperator::Gt
- | BinaryOperator::Lt
- | BinaryOperator::Gte
- | BinaryOperator::Lte
- | BinaryOperator::Lshift
- | BinaryOperator::Rshift
- | BinaryOperator::RshiftUnsigned
- | BinaryOperator::BitAnd
- | BinaryOperator::BitOr
- | BinaryOperator::BitXor
- | BinaryOperator::Mod => match (left, right) {
- (Literal::Number(a), Literal::Number(b)) => Ok(match self {
- BinaryOperator::Add => Literal::Number(a + b),
- BinaryOperator::Sub => Literal::Number(a - b),
- BinaryOperator::Mul => Literal::Number(a * b),
- BinaryOperator::Div => Literal::Number(a / b),
- BinaryOperator::Pow => Literal::Number(a.powf(b)),
- BinaryOperator::Gt => Literal::Bool(a > b),
- BinaryOperator::Lt => Literal::Bool(a < b),
- BinaryOperator::Gte => Literal::Bool(a >= b),
- BinaryOperator::Lte => Literal::Bool(a <= b),
- BinaryOperator::Lshift => Literal::Number(((a as i32) << (b as i32)).into()),
- BinaryOperator::Rshift => Literal::Number(((a as i32) >> (b as i32)).into()),
- BinaryOperator::RshiftUnsigned => {
- Literal::Number(((a as u32) >> (b as u32)).into())
- }
- BinaryOperator::BitAnd => Literal::Number(((a as i32) & (b as i32)).into()),
- BinaryOperator::BitOr => Literal::Number(((a as i32) | (b as i32)).into()),
- BinaryOperator::BitXor => Literal::Number(((a as i32) ^ (b as i32)).into()),
- BinaryOperator::Mod => Literal::Number(a % b),
- _ => unreachable!(),
- }),
- _ => Err(()),
- },
-
- // Impossible condition doing assign-add to a literal
- _ => Err(()),
- }
- }
-}
-
-impl UnaryOperator {
- pub fn to_str(&self) -> &'static str {
- match self {
- Self::Neg => "-",
- Self::Not => "!",
- Self::BitNot => "~",
- }
- }
-
- pub fn to_string(&self) -> String {
- String::from(self.to_str())
- }
-
- pub fn perform(self, value: Literal) -> Result {
- match self {
- UnaryOperator::Neg => match value {
- Literal::Number(a) => Ok(Literal::Number(-a)),
- _ => Err(()),
- },
- UnaryOperator::Not => match value {
- Literal::Bool(a) => Ok(Literal::Bool(!a)),
- _ => Err(()),
- },
- UnaryOperator::BitNot => match value {
- Literal::Number(a) => Ok(Literal::Number((!(a as i32)).into())),
- _ => Err(()),
- },
- }
- }
-}
-
-/// Taken from from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence#table
-///
-/// Blight doesn't have all of these operators, but the
-/// printer uses it to determine where to put parens.
-#[derive(PartialEq, PartialOrd, Debug)]
-pub enum OperatorLevel {
- Accessor, // e. e[] e()
- New, // new
- Postfix, // expr++ expr--
- Prefix, // ++e, --e, !e, ~e, +e, -e, typeof, void, delete
-
- // these are the binary operators
- Exponentiation, // **
- Multiplicative, // *, /, %
- Additive, // +, -
- BitwiseShift, // <<, >>, >>>
- Relational, // <, <=, >, >=
- Equality, // ==, !=
- BitwiseAnd, // &
- BitwiseXor, // ^
- BitwiseOr, // |
- LogicalAnd, // &&
- LogicalOr, // ||, ??
-
- AssignmentAndOther, // =, +=, -=, *=, /=, %=, / <<=, >>=, >>>=, &=, ^=, |=
- Comma, // ,
-}
-
-#[derive(PartialEq)]
-pub enum Associativity {
- LeftToRight,
- RightToLeft,
-}
-
-impl OperatorLevel {
- pub fn associativity(&self) -> Associativity {
- use Associativity::*;
- use OperatorLevel::*;
-
- match self {
- Accessor => LeftToRight,
- Exponentiation => RightToLeft,
- Multiplicative => LeftToRight,
- Additive => LeftToRight,
- BitwiseShift => LeftToRight,
- Relational => LeftToRight,
- Equality => LeftToRight,
- BitwiseAnd => LeftToRight,
- BitwiseXor => LeftToRight,
- BitwiseOr => LeftToRight,
- LogicalAnd => LeftToRight,
- LogicalOr => LeftToRight,
- AssignmentAndOther => RightToLeft,
- Comma => LeftToRight,
-
- // These do not make sense to have an associativity
- // You will likely not run into this if you do
- // BinaryOperator::level(o).associativity()
- New | Postfix | Prefix => unreachable!("{:?} does not have an associativity", self),
- }
- }
-}
-
-impl From<&BinaryOperator> for OperatorLevel {
- fn from(operator: &BinaryOperator) -> Self {
- use BinaryOperator::*;
- use OperatorLevel::*;
-
- match operator {
- Add | Sub => Additive,
- Mul | Div | Mod => Multiplicative,
- Pow => Exponentiation,
- Gt | Lt | Gte | Lte => Relational,
- Lshift | Rshift | RshiftUnsigned => BitwiseShift,
- BitAnd => BitwiseAnd,
- BitOr => BitwiseOr,
- BitXor => BitwiseXor,
- Eq | NotEq => Equality,
- And => LogicalAnd,
- Or => LogicalOr,
-
- Assign | AssignAdd | AssignSub | AssignMul | AssignDiv | AssignPow | AssignLshift
- | AssignRshift | AssignRshiftUnsigned | AssignBitAnd | AssignBitOr | AssignBitXor
- | AssignMod | AssignAnd | AssignOr => AssignmentAndOther,
- }
- }
-}
-
-impl From for OperatorLevel {
- fn from(operator: UnaryOperator) -> Self {
- use OperatorLevel::*;
- use UnaryOperator::*;
-
- match operator {
- Neg | Not | BitNot => Prefix,
- }
- }
-}
diff --git a/crates/ast/types.rs b/crates/ast/types.rs
deleted file mode 100644
index d1afb35a3848d2331616da58398410b440a2af6c..0000000000000000000000000000000000000000
--- a/crates/ast/types.rs
+++ /dev/null
@@ -1,134 +0,0 @@
-use miette::{Diagnostic, Result};
-use thiserror::Error;
-
-/// A value that is known at compile-time
-#[derive(Debug, Clone, PartialEq)]
-pub enum Literal {
- Number(f64),
- Bool(bool),
- String(String),
- Void,
-}
-
-impl std::fmt::Display for Literal {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- match self {
- Literal::Number(value) => write!(f, "{}", value),
- Literal::Bool(value) => write!(f, "{}", value),
- Literal::String(value) => write!(f, "{}", value),
- Literal::Void => write!(f, "void"),
- }
- }
-}
-
-#[derive(Debug, Clone, PartialEq)]
-pub enum Type {
- /// Variables without an explicit type are assigned this until
- /// the type inferencer figures out what this is. There is not a
- /// keyword for this.
- Infer,
- /// The type assigned to literals. This allows integer literals
- /// to coerce to integer or float depending on the context, but
- /// prevents implicit casting of runtime values.
- Literal(Literal),
- /// Return value of `@raw`. All type checks are disabled.
- Any,
- /// The type of `void`. This is the return type of functions
- /// that do not return a value.
- Void,
- /// noreturn is the type of 'return', aka a value that never exists.
- /// unlike void, this includes functions that do not return at all.
- NoReturn,
- /// 32 signed integer
- Int,
- /// 64 signed JS float
- Float,
- /// Boolean
- Bool,
- /// JS BigInt
- BigInt,
- /// JS String
- String,
- /// A union of multiple possible types. Might be removed
- Union(Vec),
-}
-
-pub trait TypeCoerceable {
- fn to_type(&self) -> Option;
-}
-
-impl TypeCoerceable for Literal {
- fn to_type(&self) -> Option {
- Some(match self {
- Literal::Number(_) => Type::Float,
- Literal::Bool(_) => Type::Bool,
- Literal::String(_) => Type::String,
- Literal::Void => Type::Void,
- })
- }
-}
-
-impl TypeCoerceable for Type {
- fn to_type(&self) -> Option {
- Some(self.clone())
- }
-}
-
-impl std::fmt::Display for Type {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- match self {
- Type::Infer => write!(f, "infered type"),
- Type::Any => write!(f, "opaque js value"),
- Type::Literal(value) => write!(f, "{}", value),
- Type::Int => write!(f, "int"),
- Type::Float => write!(f, "float"),
- Type::BigInt => write!(f, "bigint"),
- Type::Bool => write!(f, "bool"),
- Type::String => write!(f, "string"),
- Type::NoReturn => write!(f, "noreturn"),
- Type::Void => write!(f, "void"),
- Type::Union(of) => {
- for (i, t) in of.iter().enumerate() {
- if i != 0 {
- write!(f, " | ")?;
- }
- write!(f, "{}", t)?;
- }
- Ok(())
- }
- }
- }
-}
-
-pub fn merge_types(a: Type, b: Type) -> Result {
- match a {
- Type::Infer => return Ok(b),
- Type::Any => return Ok(Type::Any),
- _ => {}
- }
-
- match b {
- Type::Infer => return Ok(a),
- Type::Any => return Ok(Type::Any),
- _ => {}
- }
-
- if a == b {
- return Ok(a);
- }
-
- Err(TypeError::new(format!("TODO: Implement type merging for {} and {}", a, b)).into())
-}
-
-#[derive(Error, Diagnostic, Debug)]
-#[error("TypeError")]
-pub struct TypeError {
- #[help]
- error: String,
-}
-
-impl TypeError {
- pub fn new(error: String) -> Self {
- Self { error }
- }
-}
diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml
deleted file mode 100644
index 1bd4c0904e6ee693867b2cde0353948f1fbb46fe..0000000000000000000000000000000000000000
--- a/crates/cli/Cargo.toml
+++ /dev/null
@@ -1,21 +0,0 @@
-[package]
-name = "cli"
-version = "0.1.0"
-edition = "2021"
-
-[[bin]]
-name = "blight"
-path = "./main.rs"
-
-[dependencies]
-clap.workspace = true
-miette.workspace = true
-thiserror.workspace = true
-
-tokenizer = { path = "../tokenizer" }
-ast = { path = "../ast" }
-parser = { path = "../parser" }
-js_ast = { path = "../js_ast" }
-js_printer = { path = "../js_printer" }
-sema = { path = "../sema" }
-log = { path = "../log" }
diff --git a/crates/cli/main.rs b/crates/cli/main.rs
deleted file mode 100644
index dfcb80febf985e7247541887ad7a59bdd65d364f..0000000000000000000000000000000000000000
--- a/crates/cli/main.rs
+++ /dev/null
@@ -1,172 +0,0 @@
-use std::{io::Write, path::PathBuf};
-
-use clap::{Parser, Subcommand};
-use miette::{diagnostic, Result};
-
-use js_printer;
-use tokenizer;
-
-#[derive(Parser)]
-#[command(author, version, about, long_about = None)]
-struct Cli {
- /// Turn debugging information on
- #[arg(short, long, action = clap::ArgAction::Count)]
- debug: u8,
-
- #[command(subcommand)]
- command: Commands,
-}
-
-#[derive(Subcommand)]
-enum Commands {
- /// Compiles a file into javascript
- Compile {
- file: PathBuf,
- // Whether to output to stdout instead of a file
- #[arg(long, default_value_t = false)]
- stdout: bool,
- /// What javascript runtime to compile to
- #[arg(short, long, default_value = "deno")]
- target: String,
- /// Format the output javascript?
- #[arg(long, default_value_t = true)]
- format: bool,
- },
- Run {
- file: PathBuf,
- /// What javascript runtime to compile to and use
- #[arg(short, long, default_value = "deno")]
- target: String,
- },
- Astdump {
- file: PathBuf,
- },
-}
-
-fn get_runtime(runtime: &String) -> String {
- match std::env::var("USER") {
- Ok(user) if user == "dave" => String::from("bun"),
- _ => runtime.clone(),
- }
-}
-
-pub fn transpile(content: String, writer: &mut dyn Write) -> Result<()> {
- // 1. Tokenize the input
- let tokens = tokenizer::tokenize(content)?;
- // 2. Parse the tokens into an AST
- let mut blight_ast = parser::parse(tokens)?;
- // 3. Semantic analysis.
- sema::analyze_module(&mut blight_ast)?;
- // 4. Optimize the AST
- // optimize::optimize_module(&mut blight_ast);
- // 5. Translate the AST into javascript
- let js_ast = js_ast::blight_to_js(blight_ast);
- // 6. Print the JS AST
- // TODO(@paperdave): use std::io::Write instead of a string
- js_printer::Print::print(&js_ast, writer)
- .map_err(|e| diagnostic!("Error while printing: {}", e))?;
- Ok(())
-}
-
-pub fn transpile_to_string(content: String) -> Result {
- let mut buffer = Vec::new();
- transpile(content, &mut buffer)?;
- Ok(String::from_utf8(buffer).unwrap())
-}
-
-pub fn astdump(content: String) -> Result {
- // 1. Tokenize the input
- let tokens = tokenizer::tokenize(content)?;
- // 2. Parse the tokens into an AST
- let blight_ast = parser::parse(tokens)?;
- return Ok(format!("{:#?}", blight_ast));
-}
-
-fn main() -> Result<()> {
- let cli = Cli::parse();
-
- match cli.command {
- Commands::Compile {
- file,
- stdout,
- target: _,
- format,
- } => {
- let contents = match std::fs::read_to_string(file.clone()) {
- Ok(contents) => contents,
- Err(e) => {
- eprintln!("Failed to read {}: {}", file.display(), e);
- std::process::exit(1);
- }
- };
-
- let mut result = transpile_to_string(contents.clone())?;
-
- if format {
- let mut formatter = std::process::Command::new("deno")
- .stdin(std::process::Stdio::piped())
- .stdout(std::process::Stdio::piped())
- .stderr(std::process::Stdio::inherit())
- .arg("fmt")
- .arg("-")
- .spawn()
- .expect("Failed to execute 'deno fmt' on output");
- let mut stdin = formatter.stdin.take().expect("Failed to open stdin");
- stdin
- .write_all(result.as_bytes())
- .expect("Failed to write to stdin");
- stdin.flush().expect("Failed to flush stdin");
- drop(stdin);
- let out = formatter
- .wait_with_output()
- .expect("Failed to execute 'deno fmt' on output");
- result = String::from_utf8(out.stdout).unwrap();
- }
-
- if stdout {
- println!("{}", result);
- } else {
- std::fs::write(file.with_extension("js"), result).unwrap();
- }
- }
- Commands::Run { file, target } => {
- let contents = std::fs::read_to_string(file.clone()).unwrap();
- let result = transpile_to_string(contents.clone())?;
- let mut temp_file = std::env::temp_dir();
- temp_file.push("temp.js");
- std::fs::write(temp_file.clone(), result).unwrap();
-
- let rt = get_runtime(&target);
-
- match rt.as_str() {
- "deno" => std::process::Command::new("deno")
- .arg("run")
- .arg("-A")
- .arg(temp_file)
- .spawn()
- .expect("Failed to execute process"),
- "node" => std::process::Command::new("node")
- .arg(temp_file)
- .spawn()
- .expect("Failed to execute process"),
- "bun" => std::process::Command::new("bun")
- .arg("run")
- .arg(temp_file)
- .spawn()
- .expect("Failed to execute process"),
- _ => {
- panic!("Unknown runtime: {}", target);
- }
- }
- .wait()
- .unwrap();
- }
- Commands::Astdump { file } => {
- let contents = std::fs::read_to_string(file.clone()).unwrap();
- let result = astdump(contents.clone())?;
- println!("{}", result);
- }
- }
-
- Ok(())
-}
diff --git a/crates/js_ast/Cargo.toml b/crates/js_ast/Cargo.toml
deleted file mode 100644
index 586042092d15e767b3355095c3459c3f209ad1ce..0000000000000000000000000000000000000000
--- a/crates/js_ast/Cargo.toml
+++ /dev/null
@@ -1,15 +0,0 @@
-[package]
-name = "js_ast"
-version = "0.1.0"
-edition = "2021"
-
-[lib]
-path = "./lib.rs"
-
-[dependencies]
-miette.workspace = true
-thiserror.workspace = true
-num.workspace = true
-
-ast = { path = "../ast" }
-log = { path = "../log" }
diff --git a/crates/js_ast/convert/block_convert_context.rs b/crates/js_ast/convert/block_convert_context.rs
deleted file mode 100644
index 01abce4a2631eabad654d1e31c99a6c8e37d4e30..0000000000000000000000000000000000000000
--- a/crates/js_ast/convert/block_convert_context.rs
+++ /dev/null
@@ -1,244 +0,0 @@
-use crate::js_ast::*;
-use ast::*;
-use log::Range;
-use std::collections::{HashMap, HashSet};
-use std::vec;
-
-use super::tree_shake_statement_list;
-
-/// A block convert context keeps track of state while converting non-valid JS expressions to JS
-pub struct BlockConvertContext {
- /// Statements in this block context
- /// This may be the merged contents of multiple blocks
- statements: Vec,
- ids: HashSet,
- next_id: u32,
- scopes: Vec,
- /// If the last item in `statements` is actually valid
- has_last_statement: bool,
-}
-
-struct VirtualScope {
- renames: HashMap,
-}
-
-pub struct TempId {
- unmapped_name: String,
-}
-
-pub struct StatementHandle {
- index: Option,
-}
-
-impl StatementHandle {
- pub fn delete(self, ctx: &mut BlockConvertContext) {
- if let Some(index) = self.index {
- ctx.statements.remove(index as usize);
- }
- }
-}
-
-impl TempId {
- pub fn get(&self, hoist_context: &BlockConvertContext) -> String {
- hoist_context.remap_identifier(&self.unmapped_name)
- }
-}
-
-impl BlockConvertContext {
- pub fn new() -> Self {
- Self {
- statements: Vec::new(),
- ids: HashSet::new(),
- next_id: 0,
- scopes: vec![VirtualScope {
- renames: HashMap::new(),
- }],
- has_last_statement: false,
- }
- }
-
- pub fn new_child(&self) -> Self {
- Self {
- statements: Vec::new(),
- ids: self.ids.clone(),
- next_id: 0,
- scopes: vec![VirtualScope {
- renames: HashMap::new(),
- }],
- has_last_statement: false,
- }
- }
-
- /// A virtual scope is a scope that exists in blight syntax
- /// but may or may not exist in blight. With these it is important to
- pub fn push_virtual_scope(&mut self) {
- self.scopes.push(VirtualScope {
- renames: HashMap::new(),
- });
- }
-
- /// See `push_virtual_scope` for more information on what this is.
- pub fn pop_virtual_scope(&mut self) {
- self.scopes.pop();
- debug_assert!(self.scopes.len() > 0);
- }
-
- pub fn generate_temp_id(&mut self, basename: &str) -> TempId {
- self.next_id += 1;
- loop {
- let name = format!("{}{}", basename, self.next_id);
- if !self.ids.contains(&name) {
- self.ids.insert(name.clone());
- return TempId {
- unmapped_name: name,
- };
- }
- self.next_id += 1;
- }
- }
-
- /// This does some extra optimizations:
- /// - `let a = b;` will not be emitted but instad write a variable alias
- pub fn add_statement(&mut self, statement: JsStatement) -> StatementHandle {
- match statement.data {
- JsStatementData::Declaration(JSDeclaration {
- name,
- init:
- Some(JsExpr {
- data: JsExprData::Identifier(init_name),
- ..
- }),
- is_constant: true,
- }) => {
- self.scopes
- .last_mut()
- .unwrap()
- .renames
- .insert(name, init_name);
- self.has_last_statement = false;
- return StatementHandle { index: None };
- }
- _ => {}
- }
- self.statements.push(statement);
- self.has_last_statement = true;
- StatementHandle {
- index: Some(self.statements.len() as u32 - 1),
- }
- }
-
- /// Get the last statement if it exists.
- /// If the statement was optimized away, this returns None
- pub fn last_statement(&self) -> Option<&JsStatement> {
- if self.has_last_statement {
- Some(self.statements.last().unwrap())
- } else {
- None
- }
- }
-
- pub fn generate_id(&mut self, basename: &String) -> TempId {
- let mut id = 0u32;
- loop {
- let name = match id {
- 0 => basename.clone(),
- _ => format!("{}{}", basename, id),
- };
- if !self.ids.contains(&name) {
- self.ids.insert(name.clone());
- return TempId {
- unmapped_name: name,
- };
- }
- id += 1;
- }
- }
-
- /// Given a blight identifier, remap it to a new identifier.
- pub fn remap_identifier(&self, name: &String) -> String {
- for scope in self.scopes.iter().rev() {
- if let Some(remapped) = scope.renames.get(name) {
- return remapped.clone();
- }
- }
- name.clone()
- }
-
- /// Declare an identifier in the current scope.
- pub fn declare_identifier(&mut self, name: String) -> TempId {
- let remapped = self.generate_id(&name);
- let scope = self.scopes.last_mut().unwrap();
- self.ids.insert(remapped.unmapped_name.clone());
- scope.renames.insert(name, remapped.unmapped_name.clone());
- remapped
- }
-
- pub fn take_last_statement(&mut self) -> Option {
- if !self.has_last_statement {
- return None;
- }
- self.has_last_statement = false;
- Some(self.statements.pop().unwrap())
- }
-
- pub fn take_last_statement_if_expr(&mut self) -> Option {
- if !self.has_last_statement {
- return None;
- }
- match self.statements.last().unwrap().data {
- JsStatementData::Expression(_) => {
- self.has_last_statement = false;
- match self.statements.pop().unwrap().data {
- JsStatementData::Expression(expr) => Some(expr),
- _ => unreachable!(),
- }
- }
- _ => return None,
- }
- }
-}
-
-pub fn new_js_assignment_statement(expr: JsExpr, target: String, range: Range) -> JsStatement {
- JsStatement::new(
- JsStatementData::Expression(new_js_assignment_expr(expr, target, range.clone())),
- range,
- None,
- )
-}
-
-pub fn new_js_assignment_expr(expr: JsExpr, target: String, range: Range) -> JsExpr {
- JsExpr::new(
- JsExprData::Binary {
- op: BinaryOperator::Assign,
- left: Box::new(JsExpr::new(
- JsExprData::Identifier(target),
- range.clone(),
- None,
- )),
- right: Box::new(expr),
- },
- range,
- None,
- )
-}
-
-impl Into for BlockConvertContext {
- fn into(self) -> JsBlock {
- JsBlock::new(self.into(), Range::EMPTY)
- }
-}
-
-impl Into for BlockConvertContext {
- fn into(self) -> JsModule {
- JsModule {
- statements: self.into(),
- }
- }
-}
-
-impl Into> for BlockConvertContext {
- fn into(mut self) -> Vec {
- tree_shake_statement_list(&mut self.statements);
- self.statements
- }
-}
diff --git a/crates/js_ast/convert/convert_valid_block.rs b/crates/js_ast/convert/convert_valid_block.rs
deleted file mode 100644
index 29ef59c7ccf294682f376ea3afe77d113592ed4f..0000000000000000000000000000000000000000
--- a/crates/js_ast/convert/convert_valid_block.rs
+++ /dev/null
@@ -1,37 +0,0 @@
-use super::convert_valid_statement_to_expr;
-use super::BlockConvertContext;
-use super::IsValidJSExpression;
-use crate::js_ast::*;
-use ast::*;
-use log::Range;
-
-pub fn convert_valid_block(ctx: &mut BlockConvertContext, block: Block, range: Range) -> JsExpr {
- debug_assert!(
- block.is_valid_js_expression(),
- "must call convert_valid_block with valid js expression"
- );
-
- let len = block.statements.len();
- let range = range;
- let block_expr = match len {
- 0 => JsExpr::new(JsExprData::Null, range.clone(), None),
- // a block expression with a single expression is just that expression
- 1 => convert_valid_statement_to_expr(ctx, block.statements.into_iter().next().unwrap()),
- // Since blocks are not valid expressions in JS, we create a
- // comma expression from a block with more than one statement.
- 2.. => {
- let mut items = Vec::new();
- for statement in block.statements.into_iter() {
- items.push(convert_valid_statement_to_expr(ctx, statement));
- }
- JsExpr::new(JsExprData::Comma { items }, range.clone(), None)
- }
- };
-
- // a block without implicit return is to be type void, so we wrap in a void expression
- if !block.has_implicit_return && len > 0 {
- JsExpr::new(JsExprData::Void(Box::new(block_expr)), range, None)
- } else {
- block_expr
- }
-}
diff --git a/crates/js_ast/convert/convert_valid_expr.rs b/crates/js_ast/convert/convert_valid_expr.rs
deleted file mode 100644
index 3e6c29e1d557d6ec835854d34225a21979192e0a..0000000000000000000000000000000000000000
--- a/crates/js_ast/convert/convert_valid_expr.rs
+++ /dev/null
@@ -1,115 +0,0 @@
-use super::convert_valid_block;
-use super::BlockConvertContext;
-use crate::convert::convert_valid_if;
-use crate::convert::process_function::process_function_to_arrow;
-use crate::convert::IsValidJSExpression;
-use crate::js_ast::*;
-use ast::*;
-
-/// Translate a blight ast expression that is a valid JS expression, to a JS expression
-/// Precondition: it is a valid JS expression, check this with `is_valid_js_expression`
-pub fn convert_valid_expr(ctx: &mut BlockConvertContext, expr: Expr) -> JsExpr {
- // Assert may be skipped in release builds as all failures are
- // annotated with a release-checked panic
- debug_assert!(
- expr.is_valid_js_expression(),
- "must call convert_valid_expr with valid js expression (use hoist instead)"
- );
-
- let range = expr.range;
- let comment = expr.comment;
-
- match expr.data {
- ExprData::Identifier(str) => JsExpr::new(
- JsExprData::Identifier(ctx.remap_identifier(&str)),
- range,
- comment,
- ),
-
- ExprData::Literal(Literal::Bool(value)) => {
- JsExpr::new(JsExprData::Bool(value), range, comment)
- }
- ExprData::Literal(Literal::Number(value)) => {
- JsExpr::new(JsExprData::Number(value), range, comment)
- }
- ExprData::Literal(Literal::String(value)) => {
- JsExpr::new(JsExprData::String(value), range, comment)
- }
- ExprData::Literal(Literal::Void) => JsExpr::new(JsExprData::Null, range, comment),
-
- ExprData::Block(block) => convert_valid_block(ctx, block, range),
-
- ExprData::Binary { op, left, right } => {
- let left = convert_valid_expr(ctx, *left);
- let right = convert_valid_expr(ctx, *right);
- JsExpr::new(
- JsExprData::Binary {
- op,
- left: Box::new(left),
- right: Box::new(right),
- },
- range,
- comment,
- )
- }
- ExprData::Unary { op, value } => {
- let value = convert_valid_expr(ctx, *value);
- JsExpr::new(
- JsExprData::Unary {
- op,
- value: Box::new(value),
- },
- range,
- comment,
- )
- }
- ExprData::Fn { body, params, .. } => {
- process_function_to_arrow(ctx, *body, params, range, comment)
- }
- ExprData::Builtin { id, arguments } => match id {
- BuiltinId::Raw => {
- let first_arg = arguments.iter().next().unwrap();
- // TODO(@paperdave): this string clone is stupid
- let str = match first_arg.data.clone() {
- ExprData::Literal(Literal::String(s)) => s,
- _ => panic!("raw expects a string"),
- };
- JsExpr::new(JsExprData::Raw(str), range, comment)
- }
- },
- ExprData::If(expr) => convert_valid_if(ctx, expr, range, comment),
-
- ExprData::Call { base, arguments } => {
- let base = convert_valid_expr(ctx, *base);
- let arguments = arguments
- .into_iter()
- .map(|x| convert_valid_expr(ctx, x))
- .collect();
- JsExpr::new(
- JsExprData::Call {
- base: Box::new(base),
- arguments,
- },
- range,
- comment,
- )
- }
-
- ExprData::Field { base, field } => JsExpr::new(
- match field.data {
- ExprData::Literal(Literal::String(name)) => JsExprData::Dot {
- base: Box::new(convert_valid_expr(ctx, *base)),
- name,
- },
- _ => JsExprData::Index {
- base: Box::new(convert_valid_expr(ctx, *base)),
- index: Box::new(convert_valid_expr(ctx, *field)),
- },
- },
- range,
- comment,
- ),
-
- ExprData::Return(_) => unreachable!("convert_valid_expr called on return"),
- }
-}
diff --git a/crates/js_ast/convert/convert_valid_if.rs b/crates/js_ast/convert/convert_valid_if.rs
deleted file mode 100644
index 0b294194266e2b189be8d7398a7bcce60a337283..0000000000000000000000000000000000000000
--- a/crates/js_ast/convert/convert_valid_if.rs
+++ /dev/null
@@ -1,38 +0,0 @@
-use super::convert_valid_block;
-use super::convert_valid_expr;
-use super::BlockConvertContext;
-use crate::convert::IsValidJSExpression;
-use crate::js_ast::*;
-use ast::*;
-use log::Range;
-
-/// Convert a valid blight if statement to a JS expression
-pub fn convert_valid_if(
- ctx: &mut BlockConvertContext,
- expr: If,
- range: Range,
- comment: Option,
-) -> JsExpr {
- debug_assert!(
- expr.is_valid_js_expression(),
- "convert_valid_if on an invalid expression (use hoist instead)"
- );
- let condition = convert_valid_expr(ctx, *expr.condition);
-
- let yes = convert_valid_block(ctx, expr.yes.block, expr.yes.range);
- let no = match expr.no {
- Some(BlockOrIf::Block(b)) => Some(convert_valid_block(ctx, b.block, b.range)),
- Some(BlockOrIf::If(else_if)) => Some(convert_valid_if(ctx, *else_if, Range::EMPTY, None)),
- None => None,
- };
-
- JsExpr::new(
- JsExprData::Ternary {
- condition: Box::new(condition),
- yes: Box::new(yes),
- no: no.map(Box::new),
- },
- range,
- comment,
- )
-}
diff --git a/crates/js_ast/convert/convert_valid_statement.rs b/crates/js_ast/convert/convert_valid_statement.rs
deleted file mode 100644
index ef0d6b5fb48c8cbe532a76df797c346fb4b915ac..0000000000000000000000000000000000000000
--- a/crates/js_ast/convert/convert_valid_statement.rs
+++ /dev/null
@@ -1,32 +0,0 @@
-use super::convert_valid_expr;
-use super::is_valid_js_expression::*;
-use super::BlockConvertContext;
-use crate::js_ast::*;
-use ast::*;
-
-/// Translate a blight ast statement that is a valid JS expression, to a JS expression
-/// Precondition: it is a valid JS expression, check this with `is_valid_js_expression`
-pub fn convert_valid_statement_to_expr(
- ctx: &mut BlockConvertContext,
- statement: Statement,
-) -> JsExpr {
- debug_assert!(
- statement.is_valid_js_expression(),
- "convert_valid_statement_to_expr on an invalid expression (use hoist instead)"
- );
-
- match statement {
- ast::Statement::Expr(expr) => convert_valid_expr(ctx, expr),
- _ => unreachable!("attempt to convert statement to js without hoisting"),
- }
-}
-
-/// Translate a blight ast statement that is a valid JS expression, to a JS statement
-/// Precondition: it is a valid JS expression, check this with `is_valid_js_expression`
-///
-/// TODO(@paperdave): Replace this with Into for JsExpr
-pub fn convert_valid_statement(ctx: &mut BlockConvertContext, statement: Statement) -> JsStatement {
- let expr = convert_valid_statement_to_expr(ctx, statement);
- let range = expr.range.clone();
- JsStatement::new(JsStatementData::Expression(expr), range, None)
-}
diff --git a/crates/js_ast/convert/hoist_block_to_expr.rs b/crates/js_ast/convert/hoist_block_to_expr.rs
deleted file mode 100644
index 16ea9fab3f641f8b3c5c3dc8e571810e8fa47f90..0000000000000000000000000000000000000000
--- a/crates/js_ast/convert/hoist_block_to_expr.rs
+++ /dev/null
@@ -1,78 +0,0 @@
-use super::is_valid_js_expression::*;
-use super::process_statement;
-use super::BlockConvertContext;
-use crate::js_ast::*;
-use ast::*;
-use log::Range;
-
-/// This block hoist is used when we have a block in an expression context.
-///
-/// All the expressions in the block are lifted above the currently
-/// processed expression, and then a temporary variable is returned.
-/// (or the last expression might get inlined as the return value)
-///
-/// A "virtual scope" is created for this block, which handles renaming
-/// situations like:
-///
-/// let x = 1;
-/// {
-/// let x = 1;
-/// }
-/// call(x)
-///
-/// The output JS of the above would be:
-///
-/// const x = 1;
-/// const x2 = 1;
-/// call(x)
-///
-/// For expressions that have branches, use `hoist_block_to_js_block`
-pub fn hoist_block_to_expr(
- ctx: &mut BlockConvertContext,
- block: Block,
- range: Range,
- comment: Option,
-) -> Option {
- debug_assert!(
- !block.is_valid_js_expression(),
- "hoist_block on a valid expression, should call convert_valid_block instead"
- );
-
- let temporary = ctx.generate_temp_id("_block");
-
- ctx.push_virtual_scope();
-
- for statement in block.statements {
- process_statement(ctx, statement);
- }
-
- ctx.pop_virtual_scope();
-
- // If the last statement is an expression, we need
- // to assign to the temporary variable.
- let result = match ctx.last_statement().map(|x| &x.data) {
- Some(JsStatementData::Expression(_)) if block.has_implicit_return => {
- match ctx.take_last_statement().unwrap().data {
- JsStatementData::Expression(expr) => expr,
- _ => unreachable!(),
- }
- }
- _ => JsExpr::new(JsExprData::Null, range.clone(), None),
- };
-
- ctx.add_statement(JsStatement::new(
- JsStatementData::Declaration(JSDeclaration {
- name: temporary.get(&ctx),
- init: Some(result),
- is_constant: false,
- }),
- range.clone(),
- comment,
- ));
-
- Some(JsExpr::new(
- JsExprData::Identifier(temporary.get(&ctx)),
- range,
- None,
- ))
-}
diff --git a/crates/js_ast/convert/hoist_block_to_js_block.rs b/crates/js_ast/convert/hoist_block_to_js_block.rs
deleted file mode 100644
index 50fe974c9b851a8d49aca0fe6fe07d550d33b8ef..0000000000000000000000000000000000000000
--- a/crates/js_ast/convert/hoist_block_to_js_block.rs
+++ /dev/null
@@ -1,70 +0,0 @@
-use crate::js_ast::*;
-use ast::*;
-use log::Range;
-
-use super::*;
-
-/// This block hoist is used when we actually need to create a JsBlock, which
-/// is needed if there is an invalid expression within a branch of code.
-///
-/// For example, `if` has two branches, the 'yes' and 'no' paths.
-/// When we cannot lower it as a ternary, we have to create two JsBlocks
-///
-/// The `target` param is a shared temporary variable that will be assigned
-/// to the result of the block.
-///
-/// Since blocks can be void or noreturn, this target is not always
-/// assigned to. The `did_assign` flag is used to indicate that.
-///
-/// For expressions that do not have branches, use `hoist_block_to_expr`
-///
-/// TODO(@paperdave): move `did_assign` into the return value
-pub fn hoist_block_to_js_block(
- ctx: &mut BlockConvertContext,
- block: Block,
- range: Range,
- target: &TempId,
- did_assign: &mut bool,
-) -> JsBlock {
- // A non-block will become a block with a single expression:
- // _temp = expr;
- //
- // We will also apply this transformation to blocks that are valid
- // expressions, because that is more concise.
- if block.is_valid_js_expression() {
- return JsBlock::new(
- vec![new_js_assignment_statement(
- convert_valid_block(ctx, block, range.clone()),
- target.get(ctx),
- range.clone(),
- )],
- range,
- );
- }
-
- let mut child = ctx.new_child();
-
- for statement in block.statements {
- if let None = process_statement(&mut child, statement) {
- return child.into();
- };
- }
-
- // If the last statement is an expression, we need
- // to assign to the temporary variable.
- if block.has_implicit_return {
- match child.take_last_statement_if_expr() {
- Some(expr) => {
- *did_assign = true;
- child.add_statement(new_js_assignment_statement(
- expr,
- target.get(ctx),
- range.clone(),
- ));
- }
- _ => {}
- }
- }
-
- child.into()
-}
diff --git a/crates/js_ast/convert/hoist_expr.rs b/crates/js_ast/convert/hoist_expr.rs
deleted file mode 100644
index 376e1072961ddfc90a8713a3cae5bb7d1d8925f2..0000000000000000000000000000000000000000
--- a/crates/js_ast/convert/hoist_expr.rs
+++ /dev/null
@@ -1,139 +0,0 @@
-use super::convert_valid_expr;
-use super::hoist_block_to_expr;
-use super::hoist_if_expr;
-use super::is_valid_js_expression::*;
-use super::BlockConvertContext;
-// use super::HasSideEffects;
-use crate::js_ast::*;
-use crate::util::is_js_identifier;
-use ast::*;
-
-/// Pushes a decl with a generated identifier to the hoisted result
-/// and returns an identifier expression for it.
-pub fn hoist_expr_if_needed(ctx: &mut BlockConvertContext, expr: Expr) -> Option {
- // if !expr.has_side_effects() {
- // return Some(ctx.hoist_expression_contents(expr)?);
- // }
-
- let init = hoist_expr_contents(ctx, expr)?;
-
- let id = ctx.generate_temp_id("_");
- let range = init.range.clone();
-
- ctx.add_statement(JsStatement::new(
- JsStatementData::Declaration(JSDeclaration {
- name: id.get(&ctx),
- init: Some(init),
- is_constant: true,
- }),
- range.clone(),
- None,
- ));
-
- Some(JsExpr::new(
- JsExprData::Identifier(id.get(&ctx)),
- range,
- None,
- ))
-}
-
-/// Returning `None` from this means that the expression is noreturn, and that
-/// the rest of the statement never occurs and can be stripped.
-pub fn hoist_expr_contents(ctx: &mut BlockConvertContext, expr: Expr) -> Option {
- if expr.is_valid_js_expression() {
- return Some(convert_valid_expr(ctx, expr));
- }
-
- match expr.data {
- ExprData::Binary { op, left, right } => {
- let left = hoist_expr_if_needed(ctx, *left)?;
- let right = hoist_expr_if_needed(ctx, *right)?;
-
- Some(JsExpr::new(
- JsExprData::Binary {
- op,
- left: Box::new(left),
- right: Box::new(right),
- },
- expr.range,
- expr.comment,
- ))
- }
-
- ExprData::Unary { op, value } => {
- let value = hoist_expr_if_needed(ctx, *value)?;
- Some(JsExpr::new(
- JsExprData::Unary {
- op,
- value: Box::new(value),
- },
- expr.range,
- expr.comment,
- ))
- }
-
- ExprData::Field { base, field } => {
- let base = hoist_expr_if_needed(ctx, *base)?;
- let field = hoist_expr_if_needed(ctx, *field)?;
-
- Some(JsExpr::new(
- match field.data {
- JsExprData::String(name) if is_js_identifier(name.as_str()) => {
- JsExprData::Dot {
- base: Box::new(base),
- name,
- }
- }
- _ => JsExprData::Index {
- base: Box::new(base),
- index: Box::new(field),
- },
- },
- expr.range,
- expr.comment,
- ))
- }
-
- ExprData::Block(block) => hoist_block_to_expr(ctx, block, expr.range, expr.comment),
-
- ExprData::If(_) => hoist_if_expr(ctx, expr),
-
- ExprData::Return(maybe_expr) => {
- let range = expr.range.clone();
- let return_value = match maybe_expr {
- Some(expr) => Some(hoist_expr_if_needed(ctx, *expr)?),
- None => None,
- };
- ctx.add_statement(JsStatement::new(
- JsStatementData::Return(return_value),
- range,
- None,
- ));
- None
- }
-
- ExprData::Call { base, arguments } => {
- let base = hoist_expr_if_needed(ctx, *base)?;
- let arguments = arguments
- .into_iter()
- .map(|x| hoist_expr_if_needed(ctx, x))
- .collect::>>()?;
-
- Some(JsExpr::new(
- JsExprData::Call {
- base: Box::new(base),
- arguments,
- },
- expr.range,
- expr.comment,
- ))
- }
-
- ExprData::Fn { .. }
- | ExprData::Builtin { .. }
- | ExprData::Literal(_)
- | ExprData::Identifier(_) => {
- unreachable!("attempt to hoist an expression that does not require it")
- }
- }
-}
diff --git a/crates/js_ast/convert/hoist_if.rs b/crates/js_ast/convert/hoist_if.rs
deleted file mode 100644
index 734d0e712bcfecf32925fedd5dcc55b8f252a7c4..0000000000000000000000000000000000000000
--- a/crates/js_ast/convert/hoist_if.rs
+++ /dev/null
@@ -1,113 +0,0 @@
-use super::convert_valid_expr;
-use super::hoist_block_to_js_block;
-use super::hoist_expr_if_needed;
-use super::BlockConvertContext;
-use super::IsValidJSExpression;
-use super::TempId;
-use crate::js_ast::*;
-use ast::*;
-use log::Range;
-
-pub fn hoist_if_expr(ctx: &mut BlockConvertContext, expr: Expr) -> Option {
- let data = match expr.data {
- ExprData::If(data) => data,
- _ => unreachable!("hoist_if_expr called on non-if expression"),
- };
-
- let temporary = ctx.generate_temp_id("_if");
-
- let decl = ctx.add_statement(JsStatement::new(
- JsStatementData::Declaration(JSDeclaration {
- name: temporary.get(&ctx),
- init: None,
- is_constant: false,
- }),
- expr.range.clone(),
- None,
- ));
-
- let condition = hoist_expr_if_needed(ctx, *data.condition)?;
- let mut did_assign = false;
-
- let js_if = hoist_if_branch(
- ctx,
- data.yes,
- data.no,
- condition,
- &mut did_assign,
- &temporary,
- );
-
- ctx.add_statement(JsStatement::new(
- JsStatementData::If(js_if),
- expr.range.clone(),
- None,
- ));
-
- if did_assign {
- Some(JsExpr::new(
- JsExprData::Identifier(temporary.get(&ctx)),
- expr.range,
- expr.comment,
- ))
- } else {
- decl.delete(ctx);
- Some(JsExpr::new(JsExprData::Null, expr.range, expr.comment))
- }
-}
-
-pub fn hoist_if_branch(
- ctx: &mut BlockConvertContext,
- yes: BlockWithRange,
- no: Option,
- condition: JsExpr,
- did_assign: &mut bool,
- temporary: &TempId,
-) -> JsIf {
- let yes = hoist_block_to_js_block(ctx, yes.block, yes.range, &temporary, did_assign);
-
- let no = match no {
- Some(BlockOrIf::Block(no)) => Some(JsElse::Block(hoist_block_to_js_block(
- ctx, no.block, no.range, &temporary, did_assign,
- ))),
- Some(BlockOrIf::If(else_if)) => {
- // If the condition of this block has a side effect, we need to
- // emit a block so that we can assign to the temporary variable.
- //
- // Otherwise, we can just emit the condition inline
- if else_if.condition.is_valid_js_expression() {
- let condition = convert_valid_expr(ctx, *else_if.condition.clone());
- Some(JsElse::If(Box::new(hoist_if_branch(
- ctx,
- else_if.yes,
- else_if.no,
- condition,
- did_assign,
- temporary,
- ))))
- } else {
- let mut temp_block = ctx.new_child();
- let condition =
- hoist_expr_if_needed(&mut temp_block, *else_if.condition.clone()).unwrap();
-
- temp_block.add_statement(JsStatement::new(
- JsStatementData::If(hoist_if_branch(
- ctx,
- else_if.yes,
- else_if.no,
- condition,
- did_assign,
- temporary,
- )),
- Range::EMPTY,
- None,
- ));
-
- Some(JsElse::Block(temp_block.into()))
- }
- }
- None => None,
- };
-
- JsIf { condition, yes, no }
-}
diff --git a/crates/js_ast/convert/is_valid_js_expression.rs b/crates/js_ast/convert/is_valid_js_expression.rs
deleted file mode 100644
index 344d817f4465cf366923d7498fbdb36034851958..0000000000000000000000000000000000000000
--- a/crates/js_ast/convert/is_valid_js_expression.rs
+++ /dev/null
@@ -1,91 +0,0 @@
-use ast::*;
-
-/// "Valid" in this context means that the expression does not contain JS statements
-/// Meaning conversion is a direct map from a blight as node into a JS one
-///
-/// A valid expression should not go through any hoist logic, but if an expression
-/// isn't valid, then it means that it must be split into multiple JS statements
-/// inside of a block.
-pub trait IsValidJSExpression {
- /// Can this be lowered to a JS ast without hoisting items
- /// ex: a `return` expression is not a valid JS expression
- fn is_valid_js_expression(&self) -> bool;
-}
-
-impl IsValidJSExpression for ExprData {
- fn is_valid_js_expression(&self) -> bool {
- match self {
- // A block containing just expressions can be an expression
- ExprData::Block(block) => block.is_valid_js_expression(),
- ExprData::If(expr) => expr.is_valid_js_expression(),
- ExprData::Return(_) => false,
- ExprData::Builtin { id, .. } => match id {
- BuiltinId::Raw => true,
- },
- ExprData::Binary { left, right, .. } => {
- left.is_valid_js_expression() && right.is_valid_js_expression()
- }
- ExprData::Unary { value, .. } => value.is_valid_js_expression(),
- ExprData::Fn { .. } => true,
- ExprData::Call { base, arguments } => {
- base.is_valid_js_expression()
- && arguments
- .iter()
- .all(|argument| argument.is_valid_js_expression())
- }
- ExprData::Field { base, field } => {
- base.is_valid_js_expression() && field.is_valid_js_expression()
- }
- ExprData::Literal(_) | ExprData::Identifier(_) => true,
- }
- }
-}
-
-impl IsValidJSExpression for Block {
- fn is_valid_js_expression(&self) -> bool {
- self.statements
- .iter()
- .all(|statement| statement.is_valid_js_expression())
- }
-}
-
-impl IsValidJSExpression for Expr {
- fn is_valid_js_expression(&self) -> bool {
- self.data.is_valid_js_expression()
- }
-}
-
-impl IsValidJSExpression for If {
- fn is_valid_js_expression(&self) -> bool {
- self.condition.is_valid_js_expression()
- && self.yes.is_valid_js_expression()
- && match &self.no {
- Some(no) => no.is_valid_js_expression(),
- None => true,
- }
- }
-}
-
-impl IsValidJSExpression for BlockOrIf {
- fn is_valid_js_expression(&self) -> bool {
- match self {
- BlockOrIf::Block(block) => block.is_valid_js_expression(),
- BlockOrIf::If(if_) => if_.is_valid_js_expression(),
- }
- }
-}
-
-impl IsValidJSExpression for BlockWithRange {
- fn is_valid_js_expression(&self) -> bool {
- self.block.is_valid_js_expression()
- }
-}
-
-impl IsValidJSExpression for Statement {
- fn is_valid_js_expression(&self) -> bool {
- match self {
- Statement::Expr(expr) => expr.data.is_valid_js_expression(),
- _ => false,
- }
- }
-}
diff --git a/crates/js_ast/convert/mod.rs b/crates/js_ast/convert/mod.rs
deleted file mode 100644
index cbcbfa9d88f9eef36ca39764b0b587b63969fdb4..0000000000000000000000000000000000000000
--- a/crates/js_ast/convert/mod.rs
+++ /dev/null
@@ -1,42 +0,0 @@
-mod block_convert_context;
-mod convert_valid_block;
-mod convert_valid_expr;
-mod convert_valid_if;
-mod convert_valid_statement;
-mod hoist_block_to_expr;
-mod hoist_block_to_js_block;
-mod hoist_expr;
-mod hoist_if;
-mod is_valid_js_expression;
-mod process_function;
-mod process_statement;
-mod side_effects;
-mod tree_shake_statement_list;
-
-pub use block_convert_context::*;
-pub use convert_valid_block::*;
-pub use convert_valid_expr::*;
-pub use convert_valid_if::*;
-pub use convert_valid_statement::*;
-pub use hoist_block_to_expr::*;
-pub use hoist_block_to_js_block::*;
-pub use hoist_expr::*;
-pub use hoist_if::*;
-pub use is_valid_js_expression::*;
-pub use process_function::*;
-pub use process_statement::*;
-// pub use side_effects::*;
-pub use tree_shake_statement_list::*;
-
-use crate::js_ast::JsModule;
-use ast::Module;
-
-pub fn blight_to_js(blight: Module) -> JsModule {
- let mut root_context = BlockConvertContext::new();
-
- for statement in blight.statements {
- process_statement(&mut root_context, statement).unwrap();
- }
-
- root_context.into()
-}
diff --git a/crates/js_ast/convert/process_function.rs b/crates/js_ast/convert/process_function.rs
deleted file mode 100644
index be0c92160efba916d2e08ff2e42ed9b07b7aa145..0000000000000000000000000000000000000000
--- a/crates/js_ast/convert/process_function.rs
+++ /dev/null
@@ -1,65 +0,0 @@
-use super::convert_valid_expr;
-use super::hoist_expr_if_needed;
-use super::process_statement;
-use super::BlockConvertContext;
-use super::IsValidJSExpression;
-use crate::js_ast::*;
-use ast::*;
-use log::Range;
-
-pub fn process_function_to_arrow(
- ctx: &mut BlockConvertContext,
- body: Expr,
- params: Vec,
- range: Range,
- comment: Option,
-) -> JsExpr {
- // A block that is a valid expression will just become
- // that expression. We can convert multiple expressions
- // into a comma expression.
- let body = if body.is_valid_js_expression() {
- JsArrowBody::Expr(convert_valid_expr(ctx, body))
- } else {
- let mut child = ctx.new_child();
-
- match body.data {
- ExprData::Block(block) => {
- for statement in block.statements {
- if let None = process_statement(&mut child, statement) {
- break;
- };
- }
-
- if block.has_implicit_return {
- if let Some(expr) = child.take_last_statement_if_expr() {
- child.add_statement(JsStatement::new(
- JsStatementData::Return(Some(expr)),
- range.clone(),
- None,
- ));
- }
- }
- }
- _ => {
- if let Some(expr) = hoist_expr_if_needed(&mut child, body) {
- child.add_statement(JsStatement::new(
- JsStatementData::Return(Some(expr)),
- range.clone(),
- None,
- ));
- }
- }
- }
-
- JsArrowBody::Block(child.into())
- };
-
- JsExpr::new(
- JsExprData::ArrowFunction {
- params: params.into_iter().map(|x| x.name).collect(),
- body: Box::new(body),
- },
- range.clone(),
- comment.map(|x| x.into()),
- )
-}
diff --git a/crates/js_ast/convert/process_statement.rs b/crates/js_ast/convert/process_statement.rs
deleted file mode 100644
index fdf3d821643d979bfab14e1f3e903aa355a2d494..0000000000000000000000000000000000000000
--- a/crates/js_ast/convert/process_statement.rs
+++ /dev/null
@@ -1,98 +0,0 @@
-use super::convert_valid_statement;
-use super::hoist_expr_contents;
-use super::process_function_to_arrow;
-use super::BlockConvertContext;
-use super::IsValidJSExpression;
-use crate::js_ast::*;
-use ast::*;
-
-pub fn process_statement(ctx: &mut BlockConvertContext, statement: Statement) -> Option<()> {
- if statement.is_valid_js_expression() {
- let result = convert_valid_statement(ctx, statement);
- ctx.add_statement(result);
- return Some(());
- }
-
- match statement {
- Statement::VarDecl(decl) => {
- // visit init before name, this way allows for:
- //
- // let a = a;
- //
- // and `a` will be looked up before it is re-declared.
- // we will also return in the noreturn case
- let init = hoist_expr_contents(ctx, decl.init)?;
- let name = ctx.declare_identifier(decl.name);
-
- let decl_data = JSDeclaration {
- name: name.get(&ctx),
- init: Some(init),
- is_constant: decl.is_constant,
- };
- let stmt = JsStatement::new(
- if decl.is_pub {
- JsStatementData::Export(JsExport::Declaration(Box::new(decl_data)))
- } else {
- JsStatementData::Declaration(decl_data)
- },
- decl.range,
- decl.comment.map(|x| x.into()),
- );
-
- ctx.add_statement(stmt);
-
- Some(())
- }
- Statement::Expr(expr) => {
- if let Some(expr) = hoist_expr_contents(ctx, expr) {
- let range = expr.range.clone();
- ctx.add_statement(JsStatement::new(
- JsStatementData::Expression(expr),
- range,
- None,
- ));
- Some(())
- } else {
- None
- }
- }
- Statement::FnDecl(FnDecl {
- name,
- params,
- body,
- range,
- comment,
- is_pub,
- ..
- }) => {
- let name = ctx.declare_identifier(name);
-
- let processed = Some(process_function_to_arrow(
- ctx,
- body,
- params,
- range.clone(),
- None,
- ));
-
- let decl_data = JSDeclaration {
- name: name.get(&ctx),
- init: processed,
- is_constant: true,
- };
-
- ctx.add_statement(JsStatement::new(
- if is_pub {
- JsStatementData::Export(JsExport::Declaration(Box::new(decl_data)))
- } else {
- JsStatementData::Declaration(decl_data)
- },
- range,
- comment.map(|x| x.into()),
- ));
-
- Some(())
- }
- Statement::Comment(_) => Some(()),
- }
-}
diff --git a/crates/js_ast/convert/side_effects.rs b/crates/js_ast/convert/side_effects.rs
deleted file mode 100644
index b66d121fa925d72b2ee65a066da6915b7a644fc5..0000000000000000000000000000000000000000
--- a/crates/js_ast/convert/side_effects.rs
+++ /dev/null
@@ -1,91 +0,0 @@
-use ast::*;
-
-pub trait HasSideEffects {
- /// Can this item be removed from the AST without changing the program's behavior?
- fn has_side_effects(&self) -> bool;
-}
-
-impl HasSideEffects for ExprData {
- fn has_side_effects(&self) -> bool {
- match self {
- ExprData::Literal(_) => false,
- ExprData::Identifier(_) => false,
-
- ExprData::Block(block) => block.has_side_effects(),
- ExprData::Binary { left, right, .. } => {
- left.has_side_effects() || right.has_side_effects()
- }
- ExprData::Unary { value, .. } => value.has_side_effects(),
- ExprData::If(ifexpr) => ifexpr.has_side_effects(),
-
- // note: a function *declaration* has no side effects, but the call *can*
- ExprData::Fn { .. } => false,
-
- // TODO(@paperdave): implement detection of pure functions
- ExprData::Call { base, arguments } => {
- base.has_side_effects() || arguments.iter().any(|x| x.has_side_effects())
- }
-
- // note: we assume field access has side effects, ie: a getter doesn't throw
- ExprData::Field { base, field } => base.has_side_effects() || field.has_side_effects(),
-
- // note: we assume the code in @raw always has side effects
- ExprData::Builtin { id, .. } => match id {
- BuiltinId::Raw => true,
- },
-
- ExprData::Return(_) => true,
- }
- }
-}
-
-impl HasSideEffects for Statement {
- fn has_side_effects(&self) -> bool {
- match self {
- Statement::Expr(expr) => expr.has_side_effects(),
- Statement::VarDecl(decl) => decl.init.has_side_effects(),
- Statement::Comment(_) => false,
- Statement::FnDecl(_) => false,
- }
- }
-}
-
-impl HasSideEffects for Block {
- fn has_side_effects(&self) -> bool {
- self.statements
- .iter()
- .any(|statement| statement.has_side_effects())
- }
-}
-
-impl HasSideEffects for Expr {
- fn has_side_effects(&self) -> bool {
- self.data.has_side_effects()
- }
-}
-
-impl HasSideEffects for If {
- fn has_side_effects(&self) -> bool {
- self.condition.has_side_effects()
- || self.yes.has_side_effects()
- || match &self.no {
- Some(no) => no.has_side_effects(),
- None => false,
- }
- }
-}
-
-impl HasSideEffects for BlockOrIf {
- fn has_side_effects(&self) -> bool {
- match self {
- BlockOrIf::Block(block) => block.has_side_effects(),
- BlockOrIf::If(if_) => if_.has_side_effects(),
- }
- }
-}
-
-impl HasSideEffects for BlockWithRange {
- fn has_side_effects(&self) -> bool {
- self.block.has_side_effects()
- }
-}
diff --git a/crates/js_ast/convert/tree_shake_statement_list.rs b/crates/js_ast/convert/tree_shake_statement_list.rs
deleted file mode 100644
index d7a631e29c443f71ddfe2409eb400df6ebe534cc..0000000000000000000000000000000000000000
--- a/crates/js_ast/convert/tree_shake_statement_list.rs
+++ /dev/null
@@ -1,31 +0,0 @@
-use crate::js_ast::*;
-
-/// Removes statements that are not needed from a JS block. Very limited.
-///
-/// BlockConvertContext will automatically execute this when converting
-/// itself into a Statement list or other data structure.
-///
-/// What this does:
-/// - Remove discarded literals `null` -> ``
-pub fn tree_shake_statement_list(list: &mut Vec) {
- let mut set_i = 0;
- for i in 0..list.len() {
- let stmt = list.get(i).unwrap();
- let should_keep = match &stmt.data {
- JsStatementData::Expression(expr) => !expr.is_literal(),
- JsStatementData::Return(_) => true,
- JsStatementData::Export(_) => true,
- JsStatementData::Declaration(_) => true,
- JsStatementData::Function(_) => true,
- JsStatementData::If(_) => true,
- };
-
- if should_keep {
- if set_i != i {
- list.swap(i, set_i)
- }
- set_i += 1;
- }
- }
- list.truncate(set_i)
-}
diff --git a/crates/js_ast/js_ast.rs b/crates/js_ast/js_ast.rs
deleted file mode 100644
index ab09c672a4735e4eb6dd4138c0b92ba2089ee298..0000000000000000000000000000000000000000
--- a/crates/js_ast/js_ast.rs
+++ /dev/null
@@ -1,198 +0,0 @@
-use ast::{BinaryOperator, Comment, UnaryOperator};
-use log::Range;
-use num::bigint::BigInt;
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct JsExpr {
- pub data: JsExprData,
- pub range: Range,
- pub comment: Option,
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct JsKV {
- pub key: JsExpr,
- pub value: JsExpr,
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum JsExprData {
- /// node emitted by the '@raw' building
- Raw(String),
- // Litearls
- Number(f64),
- BigInt(BigInt),
- String(String),
- Bool(bool),
- Array(Vec),
- Object(Vec),
- Null,
- Undefined,
-
- Identifier(String),
- Unary {
- op: UnaryOperator,
- value: Box,
- },
- Binary {
- op: BinaryOperator,
- left: Box,
- right: Box,
- },
- Dot {
- base: Box,
- name: String,
- },
- Index {
- base: Box,
- index: Box,
- },
- Call {
- base: Box,
- arguments: Vec,
- },
- Function {
- name: Option,
- params: Vec,
- body: Box,
- },
- ArrowFunction {
- params: Vec,
- body: Box,
- },
- Ternary {
- condition: Box,
- yes: Box,
- no: Option>,
- },
- Comma {
- items: Vec,
- },
- Void(Box),
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct JsStatement {
- pub data: JsStatementData,
- pub range: Range,
- pub comment: Option,
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct JSFunctionDeclaration {
- pub name: String,
- pub params: Vec,
- pub block: JsBlock,
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct JSDeclaration {
- pub name: String,
- pub init: Option,
- pub is_constant: bool,
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct JsIf {
- pub condition: JsExpr,
- pub yes: JsBlock,
- pub no: Option,
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum JsElse {
- Expr(JsExpr),
- Block(JsBlock),
- If(Box),
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum JsStatementData {
- Expression(JsExpr),
- Return(Option),
- Export(JsExport),
- Declaration(JSDeclaration),
- Function(JSFunctionDeclaration),
- If(JsIf),
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum JsExport {
- Named(Vec),
- Function(Box),
- Declaration(Box),
- Default(JsExpr),
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct JSNamedExport {
- pub identifier: String,
- pub export_name: String,
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct JsBlock {
- pub statements: Vec,
- pub range: Range,
-}
-
-impl JsExpr {
- pub fn new(data: JsExprData, range: Range, comment: Option) -> Self {
- Self {
- data,
- range,
- comment,
- }
- }
-
- pub fn is_literal(&self) -> bool {
- match &self.data {
- JsExprData::Number(_) => true,
- JsExprData::BigInt(_) => true,
- JsExprData::String(_) => true,
- JsExprData::Bool(_) => true,
- JsExprData::Null => true,
- JsExprData::Undefined => true,
- _ => false,
- }
- }
-}
-
-impl JsExprData {
- /// Helper for testing
- pub fn str(value: &str) -> Self {
- Self::String(value.to_string())
- }
-
- /// Helper for testing
- pub fn str_id(value: &str) -> Self {
- Self::Identifier(value.to_string())
- }
-}
-
-impl JsBlock {
- pub fn new(statements: Vec, range: Range) -> Self {
- Self { statements, range }
- }
-}
-
-impl JsStatement {
- pub fn new(data: JsStatementData, range: Range, comment: Option) -> Self {
- Self {
- data,
- range,
- comment,
- }
- }
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum JsArrowBody {
- Block(JsBlock),
- Expr(JsExpr),
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct JsModule {
- pub statements: Vec,
-}
diff --git a/crates/js_ast/lib.rs b/crates/js_ast/lib.rs
deleted file mode 100644
index 9d29be2c29a04ae7f9bc7701413e57ec70196e62..0000000000000000000000000000000000000000
--- a/crates/js_ast/lib.rs
+++ /dev/null
@@ -1,7 +0,0 @@
-mod convert;
-mod js_ast;
-mod util;
-
-pub use convert::*;
-pub use js_ast::*;
-pub use util::*;
diff --git a/crates/js_ast/util.rs b/crates/js_ast/util.rs
deleted file mode 100644
index 234ee189e91921b0e25bcc7a1e717ca485459d4c..0000000000000000000000000000000000000000
--- a/crates/js_ast/util.rs
+++ /dev/null
@@ -1,17 +0,0 @@
-/// TODO(@paperdave): handle the rest of the identifiers
-pub fn is_js_identifier(s: &str) -> bool {
- if s.is_empty() {
- return false;
- }
- let mut chars = s.chars();
- let first = chars.next().unwrap();
- if !first.is_ascii_alphabetic() && first != '_' {
- return false;
- }
- for c in chars {
- if !c.is_ascii_alphanumeric() && c != '_' {
- return false;
- }
- }
- true
-}
diff --git a/crates/js_printer/Cargo.toml b/crates/js_printer/Cargo.toml
deleted file mode 100644
index 785143495163bac04e1b4dee49fe808c70aa7d15..0000000000000000000000000000000000000000
--- a/crates/js_printer/Cargo.toml
+++ /dev/null
@@ -1,14 +0,0 @@
-[package]
-name = "js_printer"
-version = "0.1.0"
-edition = "2021"
-
-[lib]
-path = "./lib.rs"
-
-[dependencies]
-miette.workspace = true
-thiserror.workspace = true
-num.workspace = true
-
-js_ast = { path = "../js_ast" }
diff --git a/crates/js_printer/lib.rs b/crates/js_printer/lib.rs
deleted file mode 100644
index 82678a2bbc537625748357d1f623957181a99c1f..0000000000000000000000000000000000000000
--- a/crates/js_printer/lib.rs
+++ /dev/null
@@ -1,377 +0,0 @@
-mod printer_tests;
-
-use js_ast::*;
-use std::io::{Result, Write};
-
-pub trait Print {
- fn print(&self, writer: &mut dyn Write) -> Result<()>;
-
- fn print_to_string(&self) -> String {
- let mut buffer = Vec::new();
- self.print(&mut buffer).unwrap();
- String::from_utf8(buffer).unwrap()
- }
-}
-
-// the printer:
-
-fn print_module_clause_identifier(writer: &mut dyn Write, export_name: &str) -> Result<()> {
- // TODO(@paperdave): Escape invalid identifiers
- if js_ast::is_js_identifier(export_name) {
- write!(writer, "{}", export_name)
- } else {
- todo!("call JSON.stringify on identifier")
- }
-}
-
-fn print_js_string(writer: &mut dyn Write, string: &str) -> Result<()> {
- write!(writer, "\"")?;
- // TODO(@paperdave): Instead of writing each character, this should write
- // chunks between the escapes instead of calling write! on each character individually.
- for c in string.chars() {
- match c {
- '\n' => write!(writer, "\\n")?,
- '\r' => write!(writer, "\\r")?,
- '\t' => write!(writer, "\\t")?,
- '"' => write!(writer, "\\\"")?,
- '\\' => write!(writer, "\\\\")?,
- '\u{0008}' => write!(writer, "\\b")?,
- '\u{000C}' => write!(writer, "\\f")?,
- '\u{0000}'..='\u{001F}' => write!(writer, "\\u{:04x}", c as u32)?,
- // TODO(@paperdave): I think there are some larger code points that need to be escaped?
- _ => write!(writer, "{}", c)?,
- }
- }
- write!(writer, "\"")
-}
-
-impl Print for JsStatement {
- fn print(&self, w: &mut dyn Write) -> Result<()> {
- match &self.data {
- js_ast::JsStatementData::Expression(expr) => {
- expr.print(w)?;
- write!(w, ";")
- }
- js_ast::JsStatementData::Return(expr) => {
- write!(w, "return")?;
- if let Some(expr) = expr {
- write!(w, " ")?;
- expr.print(w)?;
- }
- write!(w, ";")?;
- Ok(())
- }
- js_ast::JsStatementData::Export(export) => export.print(w),
- js_ast::JsStatementData::Declaration(decl) => decl.print(w),
- js_ast::JsStatementData::Function(function) => function.print(w),
- js_ast::JsStatementData::If(js_if) => js_if.print(w),
- }
- }
-}
-
-impl Print for JsExport {
- fn print(&self, w: &mut dyn Write) -> Result<()> {
- match self {
- JsExport::Named(exports) => {
- write!(w, "export{{")?;
- for (i, export) in exports.iter().enumerate() {
- if i > 0 {
- write!(w, ",")?;
- }
- export.print(w)?;
- }
- Ok(())
- }
- JsExport::Function(function) => {
- write!(w, "export ")?;
- function.print(w)
- }
- JsExport::Declaration(declaration) => {
- write!(w, "export ")?;
- declaration.print(w)
- }
- JsExport::Default(expr) => {
- write!(w, "export default ")?;
- expr.print(w)
- }
- }
- }
-}
-
-impl Print for JSNamedExport {
- fn print(&self, writer: &mut dyn Write) -> Result<()> {
- if self.export_name == self.identifier {
- write!(writer, "{}", self.identifier)
- } else {
- write!(writer, "{} as ", self.identifier)?;
- print_module_clause_identifier(writer, &self.export_name)
- }
- }
-}
-
-impl Print for JSDeclaration {
- fn print(&self, writer: &mut dyn Write) -> Result<()> {
- write!(
- writer,
- "{} {}",
- if self.is_constant { "const" } else { "let" },
- self.name,
- )?;
- if let Some(init) = &self.init {
- write!(writer, " = ")?;
- init.print(writer)?;
- }
- write!(writer, ";")?;
- Ok(())
- }
-}
-
-impl Print for JsExpr {
- fn print(&self, writer: &mut dyn Write) -> Result<()> {
- self.data.print(writer)
- }
-}
-
-impl Print for JsExprData {
- fn print(&self, writer: &mut dyn Write) -> Result<()> {
- match &self {
- JsExprData::Raw(data) => write!(writer, "{}", data),
- JsExprData::Number(number) => write!(writer, "{}", number),
- JsExprData::BigInt(bigint) => write!(writer, "{}", bigint),
- JsExprData::String(string) => print_js_string(writer, string.as_str()),
- JsExprData::Bool(bool) => write!(writer, "{}", bool),
- JsExprData::Null => write!(writer, "null"),
- JsExprData::Undefined => write!(writer, "undefined"),
- JsExprData::Identifier(str) => write!(writer, "{}", str),
- JsExprData::Array(items) => {
- write!(writer, "[")?;
- for (i, item) in items.iter().enumerate() {
- if i > 0 {
- write!(writer, ",")?;
- }
- item.print(writer)?;
- }
- write!(writer, "]")
- }
- JsExprData::Object(items) => {
- write!(writer, "{{")?;
- for (i, kv) in items.iter().enumerate() {
- if i > 0 {
- write!(writer, ",")?;
- }
-
- kv.print(writer)?;
- }
- write!(writer, "}}")
- }
- JsExprData::Unary { op, value } => {
- write!(writer, "{}", op.to_str())?;
- value.print(writer)
- }
- JsExprData::Binary { op, left, right } => {
- // TODO: figure out when we dont need to wrap
- let wrap = true;
- if wrap {
- write!(writer, "(")?;
- }
- left.print(writer)?;
- write!(writer, "{}", op.to_str())?;
- right.print(writer)?;
- if wrap {
- write!(writer, ")")?;
- }
- Ok(())
- }
- JsExprData::Dot { base, name } => {
- base.print(writer)?;
- write!(writer, ".{}", name)
- }
- JsExprData::Index { base, index } => {
- base.print(writer)?;
- write!(writer, "[")?;
- index.print(writer)?;
- write!(writer, "]")
- }
- JsExprData::Call { base, arguments } => {
- base.print(writer)?;
- write!(writer, "(")?;
- for (i, argument) in arguments.iter().enumerate() {
- if i > 0 {
- write!(writer, ",")?;
- }
- argument.print(writer)?;
- }
- write!(writer, ")")
- }
- JsExprData::Function { body, name, params } => {
- write!(writer, "function ")?;
- if let Some(name) = name {
- write!(writer, "{}", name)?;
- }
- write!(writer, "(")?;
- for (i, param) in params.iter().enumerate() {
- if i > 0 {
- write!(writer, ",")?;
- }
- write!(writer, "{}", param)?;
- }
- write!(writer, ")")?;
- body.print(writer)
- }
- JsExprData::ArrowFunction { body, params } => {
- let print_parens = params.len() != 1;
- if print_parens {
- write!(writer, "(")?;
- }
- for (i, param) in params.iter().enumerate() {
- if i > 0 {
- write!(writer, ",")?;
- }
- write!(writer, "{}", param)?;
- }
- if print_parens {
- write!(writer, ")")?;
- }
- write!(writer, "=>")?;
- body.print(writer)
- }
- JsExprData::Ternary { condition, yes, no } => {
- let wrap = false;
- if wrap {
- write!(writer, "(")?;
- }
- condition.print(writer)?;
- write!(writer, "?")?;
- yes.print(writer)?;
- write!(writer, ":")?;
- if let Some(no) = no {
- no.print(writer)?;
- } else {
- write!(writer, "null")?;
- }
- if wrap {
- write!(writer, ")")?;
- }
- Ok(())
- }
- JsExprData::Comma { items } => {
- let wrap = true;
- if wrap {
- write!(writer, "(")?;
- }
- for (i, item) in items.iter().enumerate() {
- if i > 0 {
- write!(writer, ",")?;
- }
- item.print(writer)?;
- }
- if wrap {
- write!(writer, ")")?;
- }
- Ok(())
- }
- JsExprData::Void(expr) => {
- write!(writer, "void ")?;
- expr.print(writer)
- }
- }
- }
-}
-
-impl Print for JSFunctionDeclaration {
- fn print(&self, writer: &mut dyn Write) -> Result<()> {
- write!(writer, "function {}", self.name)?;
- write!(writer, "(")?;
- for (i, param) in self.params.iter().enumerate() {
- if i > 0 {
- write!(writer, ",")?;
- }
- write!(writer, "{}", param)?;
- }
- write!(writer, ")")?;
- self.block.print(writer)
- }
-}
-
-impl Print for JsBlock {
- fn print(&self, writer: &mut dyn Write) -> Result<()> {
- write!(writer, "{{")?;
- for statement in self.statements.iter() {
- statement.print(writer)?;
- }
- write!(writer, "}}")
- }
-}
-
-impl Print for JsIf {
- fn print(&self, writer: &mut dyn Write) -> Result<()> {
- write!(writer, "if(")?;
- self.condition.print(writer)?;
- write!(writer, ")")?;
- self.yes.print(writer)?;
- if let Some(no) = &self.no {
- no.print(writer)?;
- }
- Ok(())
- }
-}
-
-impl Print for JsElse {
- fn print(&self, writer: &mut dyn Write) -> Result<()> {
- match self {
- JsElse::If(js_if) => {
- write!(writer, "else ")?;
- js_if.print(writer)
- }
- JsElse::Block(block) => {
- write!(writer, "else")?;
- block.print(writer)
- }
- JsElse::Expr(expr) => {
- write!(writer, "else ")?;
- expr.print(writer)
- }
- }
- }
-}
-
-impl Print for JsKV {
- fn print(&self, writer: &mut dyn Write) -> Result<()> {
- match &self.key.data {
- JsExprData::String(string) => {
- if is_js_identifier(string) {
- write!(writer, "{}", string)?;
- } else {
- print_js_string(writer, string.as_str())?;
- }
- }
- _ => {
- write!(writer, "[")?;
- self.key.print(writer)?;
- write!(writer, "]")?;
- }
- }
- write!(writer, ":")?;
- self.key.print(writer)?;
- write!(writer, ":")?;
- self.value.print(writer)
- }
-}
-
-impl Print for JsModule {
- fn print(&self, writer: &mut dyn Write) -> Result<()> {
- for statement in self.statements.iter() {
- statement.print(writer)?;
- }
- Ok(())
- }
-}
-
-impl Print for JsArrowBody {
- fn print(&self, writer: &mut dyn Write) -> Result<()> {
- match self {
- JsArrowBody::Block(block) => block.print(writer),
- JsArrowBody::Expr(expr) => expr.print(writer),
- }
- }
-}
diff --git a/crates/js_printer/printer_tests.rs b/crates/js_printer/printer_tests.rs
deleted file mode 100644
index d69ab247dffe40609ed2892eca79fedb3c4f1a3a..0000000000000000000000000000000000000000
--- a/crates/js_printer/printer_tests.rs
+++ /dev/null
@@ -1,53 +0,0 @@
-#![cfg(test)]
-use crate::Print;
-
-use js_ast::JsExprData as E;
-use num::{bigint::Sign, BigInt};
-
-macro_rules! assert_printed {
- ($value:expr, $expected:expr) => {
- assert_eq!($value.print_to_string(), $expected);
- };
-}
-
-#[test]
-fn number() {
- assert_printed!(E::Number(1.0), "1");
- assert_printed!(E::Number(1.5), "1.5");
- assert_printed!(E::Number(1.5e10), "15000000000");
- assert_printed!(E::Number(1.5e-10), "0.00000000015");
- assert_printed!(E::Number(-1.0), "-1");
- assert_printed!(E::Number(-1.5), "-1.5");
- assert_printed!(E::Number(0f64), "0");
-}
-
-#[test]
-fn bigint() {
- assert_printed!(E::BigInt(BigInt::new(Sign::Plus, vec![1])), "1n");
- assert_printed!(
- E::BigInt(BigInt::new(Sign::Plus, vec![10000000])),
- "10000000n"
- );
- assert_printed!(
- E::BigInt(BigInt::new(Sign::Minus, vec![10000000])),
- "-10000000n"
- );
-}
-
-#[test]
-fn boolean() {
- assert_printed!(E::Bool(true), "true");
- assert_printed!(E::Bool(false), "false");
-}
-
-#[test]
-fn string() {
- assert_printed!(E::str(""), "\"\"");
- assert_printed!(E::str("hello"), "\"hello\"");
- assert_printed!(E::str("hello\nworld"), "\"hello\\nworld\"");
-}
-
-#[test]
-fn identifier() {
- assert_printed!(E::str_id("hello"), "hello");
-}
diff --git a/crates/log/Cargo.toml b/crates/log/Cargo.toml
deleted file mode 100644
index 1ec8e700cac532f571aea79f215f0ac610e6d9e4..0000000000000000000000000000000000000000
--- a/crates/log/Cargo.toml
+++ /dev/null
@@ -1,11 +0,0 @@
-[package]
-name = "log"
-edition = "2021"
-
-[lib]
-path="./log.rs"
-
-[dependencies]
-miette.workspace = true
-thiserror.workspace = true
-url.workspace = true
diff --git a/crates/log/log.rs b/crates/log/log.rs
deleted file mode 100644
index 7d91cbbbdb54e2058e2807f3c699a48532d458b4..0000000000000000000000000000000000000000
--- a/crates/log/log.rs
+++ /dev/null
@@ -1,88 +0,0 @@
-use std::{fs, path::Path, rc::Rc};
-
-use miette::{IntoDiagnostic, Result};
-use url::Url;
-
-#[derive(Debug, PartialEq, Clone, Copy)]
-pub struct Range {
- pub start: usize,
- pub length: usize,
-}
-
-impl Range {
- /// TODO(@paperdave): Remove this. every usage of this is incorrect.
- pub const EMPTY: Range = Range {
- start: 0,
- length: 0,
- };
-
- pub fn new(start: usize, length: usize) -> Self {
- Self { start, length }
- }
-
- pub fn len(&self) -> usize {
- self.length
- }
-}
-
-// TODO(@lino-levan): make this loggable nicely
-#[derive(Debug, Clone)]
-pub struct Message<'a> {
- pub source: &'a Source,
- pub range: Range,
- pub message: String,
- pub notes: Vec>,
-}
-
-#[derive(Debug, Clone)]
-pub struct Note<'a> {
- pub location: Option>,
- pub message: String,
-}
-
-#[derive(Debug, Clone)]
-pub struct RangeAndSource<'a> {
- pub range: Range,
- pub source: &'a Source,
-}
-
-#[derive(Debug)]
-pub struct Source {
- pub url: Url,
- pub code: String,
-}
-
-impl Source {
- pub fn new(url: Url, source_code: String) -> Rc {
- Rc::new(Source {
- url,
- code: source_code,
- })
- }
-
- pub fn from_file(path: &Path) -> Result> {
- let source_code = fs::read_to_string(path).into_diagnostic()?;
- let url = Url::from_file_path(path).unwrap();
- Ok(Rc::new(Source {
- code: source_code,
- url,
- }))
- }
-
- pub fn from_url(url: Url) -> Result> {
- match url.scheme() {
- "file" => {
- // TODO: how to return this error, into_diagnostic() doesn't exist
- let file_path = url.to_file_path().unwrap();
- let source_code = fs::read_to_string(file_path).into_diagnostic()?;
- Ok(Rc::new(Source {
- code: source_code,
- url,
- }))
- }
- _ => {
- todo!("Fetch from URL: {}", url)
- }
- }
- }
-}
diff --git a/crates/parser/Cargo.toml b/crates/parser/Cargo.toml
deleted file mode 100644
index 0a7ff98b649756ba9871e3dd13f63cfcc6abd6a5..0000000000000000000000000000000000000000
--- a/crates/parser/Cargo.toml
+++ /dev/null
@@ -1,16 +0,0 @@
-[package]
-name = "parser"
-version = "0.1.0"
-edition = "2021"
-
-[lib]
-path = "./parser.rs"
-
-[dependencies]
-miette.workspace = true
-thiserror.workspace = true
-num.workspace = true
-
-ast = { path = "../ast" }
-tokenizer = { path = "../tokenizer" }
-log = { path = "../log" }
diff --git a/crates/parser/parse_block.rs b/crates/parser/parse_block.rs
deleted file mode 100644
index a7b827a93fb636a6e34409a1e6d45e05e1578aab..0000000000000000000000000000000000000000
--- a/crates/parser/parse_block.rs
+++ /dev/null
@@ -1,84 +0,0 @@
-use crate::{parse_statement::parse_statement, Parser};
-use ast::Block;
-use miette::Result;
-use tokenizer::TokenType;
-
-/// Precondition: The first token exists and is '{'
-pub fn parse_block(p: &mut Parser) -> Result {
- match p.next() {
- Some(TokenType::OpenBrace) => (),
- _ => unreachable!(),
- }
-
- let mut statements = Vec::new();
-
- let was_top_level = p.is_top_level;
- p.is_top_level = false;
-
- 'outer: loop {
- let is_statement_like_expression = p
- .peek()
- .map_or(false, |x| x.is_start_of_statement_like_expression());
-
- match p.peek() {
- Some(TokenType::ClosedBrace) => {
- p.next();
- p.is_top_level = was_top_level;
- return Ok(Block {
- statements,
- has_implicit_return: false,
- });
- }
- _ => (),
- }
-
- let statement = match parse_statement(p) {
- Ok(statement) => statement,
- Err(err) => {
- p.is_top_level = was_top_level;
- return Err(err);
- }
- };
- statements.push(statement.clone());
-
- match p.peek() {
- Some(TokenType::Semicolon) => loop {
- p.next();
- match p.peek() {
- Some(TokenType::ClosedBrace) => {
- p.next();
- p.is_top_level = was_top_level;
- return Ok(Block {
- statements,
- has_implicit_return: false,
- });
- }
- _ => continue 'outer,
- }
- },
- Some(TokenType::ClosedBrace) => {
- p.next();
- p.is_top_level = was_top_level;
-
- if !statement.is_expression() {
- return Err(p
- .error("Expected semicolon or '}' after statement".to_string())
- .into());
- }
-
- return Ok(Block {
- statements,
- has_implicit_return: true,
- });
- }
- _ => {
- if statement.require_semicolon(is_statement_like_expression) {
- p.is_top_level = was_top_level;
- return Err(p
- .error("Expected semicolon or '}' after statement".to_string())
- .into());
- }
- }
- };
- }
-}
diff --git a/crates/parser/parse_call_arguments.rs b/crates/parser/parse_call_arguments.rs
deleted file mode 100644
index 88b709e7e3ef62f594e2e3429588ab6781cfcd84..0000000000000000000000000000000000000000
--- a/crates/parser/parse_call_arguments.rs
+++ /dev/null
@@ -1,36 +0,0 @@
-use ast::Expr;
-use miette::Result;
-use tokenizer::TokenType;
-
-use crate::{parse_expr::parse_expr, Parser};
-
-/// Precondition: The '(' was already consumed
-pub fn parse_call_arguments(p: &mut Parser) -> Result> {
- let mut args = Vec::new();
-
- if let Some(TokenType::ClosedParen) = p.peek() {
- _ = p.next();
- return Ok(args);
- }
-
- loop {
- let arg = parse_expr(p)?;
- args.push(arg);
-
- match p.peek() {
- Some(TokenType::Comma) => {
- _ = p.next();
- continue;
- }
- Some(TokenType::ClosedParen) => {
- _ = p.next();
- return Ok(args);
- }
- _ => {
- return Err(p
- .error("Expected ',' or ')' after argument".to_string())
- .into())
- }
- }
- }
-}
diff --git a/crates/parser/parse_declaration.rs b/crates/parser/parse_declaration.rs
deleted file mode 100644
index 0d5ae7e9f33c978ca796bce72edaac57b8e72e91..0000000000000000000000000000000000000000
--- a/crates/parser/parse_declaration.rs
+++ /dev/null
@@ -1,51 +0,0 @@
-use ast::{DocComment, Type, VarDecl};
-use log::Range;
-use miette::Result;
-use tokenizer::TokenType;
-
-use crate::parse_expr::parse_expr;
-use crate::parse_type::parse_type;
-use crate::Parser;
-
-/// Precondition: The first token exists and is either 'var' or 'let'
-pub fn parse_declaration(
- p: &mut Parser,
- is_pub: bool,
- comment: Option,
-) -> Result {
- let start = p.index;
- let is_constant = match p.next() {
- Some(TokenType::Var) => false,
- Some(TokenType::Let) => true,
- _ => unreachable!(),
- };
- let name = match p.next() {
- Some(TokenType::Identifier(name)) => name,
- Some(tok) => return Err(p.error(format!("Unexpected {}", tok)).into()),
- None => return Err(p.error(format!("Unexpected end of file")).into()),
- };
- let return_type = match p.peek() {
- Some(TokenType::Colon) => {
- p.next();
- parse_type(p)?
- }
- _ => Type::Infer,
- };
- let init = match p.next() {
- Some(TokenType::Equal) => parse_expr(p)?,
- _ => {
- return Err(p
- .error("Variable must have an initializer".to_string())
- .into())
- }
- };
- Ok(VarDecl {
- name: name.clone(),
- data_type: return_type,
- init,
- is_constant,
- is_pub,
- comment,
- range: Range::new(start, p.index - start),
- })
-}
diff --git a/crates/parser/parse_doc_comment.rs b/crates/parser/parse_doc_comment.rs
deleted file mode 100644
index cb138e3abb6e12f97ef8f6ec4a01d7106004c5b7..0000000000000000000000000000000000000000
--- a/crates/parser/parse_doc_comment.rs
+++ /dev/null
@@ -1,19 +0,0 @@
-use ast::DocComment;
-use log::Range;
-use tokenizer::TokenType;
-
-use crate::Parser;
-
-pub fn maybe_read_doc_comment(p: &mut Parser) -> Option {
- match p.peek() {
- Some(TokenType::DocComment(data)) => {
- let start = p.index;
- p.next();
- Some(DocComment {
- data,
- range: Range::new(start, p.index - start),
- })
- }
- _ => None,
- }
-}
diff --git a/crates/parser/parse_expr.rs b/crates/parser/parse_expr.rs
deleted file mode 100644
index f0a1ed29ef2aa43e331f7e227ab984fd39a3701b..0000000000000000000000000000000000000000
--- a/crates/parser/parse_expr.rs
+++ /dev/null
@@ -1,71 +0,0 @@
-use ast::{Associativity, BinaryOperator, Expr, ExprData, OperatorLevel};
-use log::Range;
-use miette::Result;
-
-use crate::parse_expr_basic::parse_expr_basic;
-use crate::Parser;
-
-pub fn parse_expr(p: &mut Parser) -> Result {
- let mut expr = parse_expr_basic(p)?;
-
- // Edge case for return. This has already parsed all the way to the end of the expression
- if let ExprData::Return(_) = expr.data {
- return Ok(expr);
- }
-
- // Parse operators
- loop {
- if let Some(token) = p.peek() {
- if let Some(level) = BinaryOperator::from_token(&token).map(|o| o.level()) {
- _ = p.next();
- expr =
- parse_expr_binary(p, BinaryOperator::from_token(&token).unwrap(), expr, level)?;
- continue;
- }
- }
- break;
- }
-
- Ok(expr)
-}
-
-/// Prerequisite: `p.peek()` is an operator
-fn parse_expr_binary(
- p: &mut Parser,
- op: BinaryOperator,
- left: Expr,
- level: OperatorLevel,
-) -> Result {
- let start = p.index;
- let mut right = parse_expr_basic(p)?;
-
- loop {
- if let Some(token) = p.peek() {
- if let Some(new_level) = BinaryOperator::from_token(&token).map(|o| o.level()) {
- if (level.associativity() == Associativity::LeftToRight && new_level < level)
- || (level.associativity() == Associativity::RightToLeft && new_level <= level)
- {
- _ = p.next();
- right = parse_expr_binary(
- p,
- BinaryOperator::from_token(&token).unwrap(),
- right,
- new_level,
- )?;
- continue;
- }
- }
- }
- break;
- }
-
- Ok(Expr::new(
- ExprData::Binary {
- op,
- left: Box::new(left),
- right: Box::new(right),
- },
- Range::new(start, p.index - start),
- None,
- ))
-}
diff --git a/crates/parser/parse_expr_basic.rs b/crates/parser/parse_expr_basic.rs
deleted file mode 100644
index 037b6a1066985d0f188185a3dafb9b5c1362c37c..0000000000000000000000000000000000000000
--- a/crates/parser/parse_expr_basic.rs
+++ /dev/null
@@ -1,230 +0,0 @@
-use ast::{BuiltinId, Expr, ExprData, Literal, UnaryOperator};
-use log::Range;
-use miette::Result;
-use tokenizer::TokenType;
-
-use crate::{
- parse_block::parse_block, parse_call_arguments::parse_call_arguments, parse_expr::parse_expr,
- parse_expr_if::parse_expr_if, Parser,
-};
-
-pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result> {
- let start = p.index;
- let comments = p.take_comments();
- let mut expr = match p.peek() {
- Some(TokenType::Number(number)) => {
- _ = p.next();
- Expr::new(
- ExprData::Literal(Literal::Number(number.clone())),
- Range::new(start, p.index - start),
- comments,
- )
- }
- Some(TokenType::String(string)) => {
- _ = p.next();
- Expr::new(
- ExprData::Literal(Literal::String(string.clone())),
- Range::new(start, p.index - start),
- comments,
- )
- }
- Some(TokenType::True) => {
- _ = p.next();
- Expr::new(
- ExprData::Literal(Literal::Bool(true)),
- Range::new(start, p.index - start),
- comments,
- )
- }
- Some(TokenType::False) => {
- _ = p.next();
- Expr::new(
- ExprData::Literal(Literal::Bool(false)),
- Range::new(start, p.index - start),
- comments,
- )
- }
- Some(TokenType::Void) => {
- _ = p.next();
- Expr::new(
- ExprData::Literal(Literal::Void),
- Range::new(start, p.index - start),
- comments,
- )
- }
- Some(TokenType::Identifier(name)) => {
- let identifier_start = p.index;
- _ = p.next();
- if name.starts_with("@") {
- if let Some(id) = BuiltinId::from_str(name.as_str()) {
- if let Some(TokenType::OpenParen) = p.next() {
- let arguments = parse_call_arguments(p)?;
- Expr::new(
- ExprData::Builtin { id, arguments },
- Range::new(identifier_start, p.index - identifier_start),
- comments,
- )
- } else {
- return Err(p
- .error_at(
- format!("Built-in function '{}' must be called", name),
- Range::new(identifier_start, p.index - identifier_start),
- )
- .into());
- }
- } else {
- return Err(p
- .error_at(
- format!("Non-existant built-in: '{}'", name),
- Range::new(identifier_start, p.index - identifier_start),
- )
- .into());
- }
- } else {
- Expr::new(
- ExprData::Identifier(name),
- Range::new(identifier_start, p.index - identifier_start),
- comments,
- )
- }
- }
-
- Some(TokenType::Return) => {
- if p.is_top_level {
- return Err(p
- .error("'return' cannot be used at top level".to_string())
- .into());
- } else {
- _ = p.next();
- let inner = maybe_parse_expr_basic(p)?.map(Box::new);
-
- return Ok(Some(Expr::new(
- ExprData::Return(inner),
- Range::new(start, p.index - start),
- comments,
- )));
- }
- }
-
- Some(TokenType::If) => Expr::new(
- ExprData::If(parse_expr_if(p)?),
- Range::new(start, p.index - start),
- comments,
- ),
-
- Some(TokenType::OpenBrace) => Expr::new(
- ExprData::Block(parse_block(p)?),
- Range::new(start, p.index - start),
- comments,
- ),
-
- Some(TokenType::OpenParen) => {
- _ = p.next();
- let expr = parse_expr(p)?;
- match p.next() {
- Some(TokenType::ClosedParen) => (),
- Some(token) => {
- return Err(p.error(format!("Expected ')', but got {:?}", token)).into())
- }
- None => return Err(p.error("Unexpected end of file".to_string()).into()),
- }
- expr
- }
-
- Some(t @ TokenType::Bang) | Some(t @ TokenType::Minus) | Some(t @ TokenType::Tilde) => {
- _ = p.next();
- Expr::new(
- ExprData::Unary {
- op: match t {
- TokenType::Bang => UnaryOperator::Not,
- TokenType::Minus => UnaryOperator::Neg,
- TokenType::Tilde => UnaryOperator::BitNot,
- _ => unreachable!(),
- },
- value: Box::new(parse_expr_basic(p)?),
- },
- Range::new(start, p.index - start),
- comments,
- )
- }
-
- Some(_) | None => return Ok(None),
- };
-
- // Parse function calls and field accesses
- loop {
- match p.peek() {
- Some(TokenType::Dot) => {
- let field_start = p.index;
- _ = p.next();
- let field = match p.peek() {
- Some(TokenType::Identifier(name)) => {
- let field_start = p.index;
- p.next();
- Expr::new(
- ExprData::Literal(Literal::String(name)),
- Range::new(field_start, p.index - field_start),
- None,
- )
- }
- _ => {
- return Err(p.error("Expected identifier after dot".to_string()).into());
- }
- };
- expr = Expr::new(
- ExprData::Field {
- base: Box::new(expr),
- field: Box::new(field),
- },
- Range::new(field_start, p.index - field_start),
- None,
- );
- }
- Some(TokenType::OpenParen) => {
- let expr_start = p.index;
- _ = p.next();
- let arguments = parse_call_arguments(p)?;
- expr = Expr::new(
- ExprData::Call {
- base: Box::new(expr),
- arguments,
- },
- Range::new(expr_start, p.index - expr_start),
- None,
- );
- }
- Some(TokenType::OpenBracket) => {
- let expr_start = p.index;
- _ = p.next();
- let index = parse_expr(p)?;
- if let Some(TokenType::ClosedBracket) = p.next() {
- expr = Expr::new(
- ExprData::Field {
- base: Box::new(expr),
- field: Box::new(index),
- },
- Range::new(expr_start, p.index - expr_start),
- None,
- );
- } else {
- return Err(p
- .error("Expected closing bracket after index".to_string())
- .into());
- }
- }
- _ => break,
- }
- }
-
- Ok(Some(expr))
-}
-
-pub fn parse_expr_basic(p: &mut Parser) -> Result {
- match maybe_parse_expr_basic(p)? {
- Some(expr) => Ok(expr),
- None => match p.peek() {
- Some(token) => Err(p.error(format!("Unexpected '{}'", token)).into()),
- None => Err(p.error("Unexpected end of file".to_string()).into()),
- },
- }
-}
diff --git a/crates/parser/parse_expr_if.rs b/crates/parser/parse_expr_if.rs
deleted file mode 100644
index 0bd246255677060afa1f44b0ebe100bc99b4ecc8..0000000000000000000000000000000000000000
--- a/crates/parser/parse_expr_if.rs
+++ /dev/null
@@ -1,71 +0,0 @@
-use ast::{BlockOrIf, BlockWithRange, If};
-use log::Range;
-use miette::Result;
-use tokenizer::TokenType;
-
-use crate::parse_block::parse_block;
-use crate::parse_expr::parse_expr;
-use crate::Parser;
-
-/// Precondition: the first token exists and is 'if'
-pub fn parse_expr_if(p: &mut Parser) -> Result {
- match p.next() {
- Some(TokenType::If) => (),
- _ => unreachable!(),
- }
-
- match p.peek() {
- Some(TokenType::OpenBrace) => {
- return Err(p
- .error(
- "To use a block as the condition for an if statement, wrap it in parentheses"
- .to_string(),
- )
- .into());
- }
- _ => (),
- }
-
- let condition = Box::new(parse_expr(p)?);
-
- let yes = match p.peek() {
- Some(TokenType::OpenBrace) => {
- let block_start = p.index;
- BlockWithRange {
- block: parse_block(p)?,
- range: Range::new(block_start, p.index - block_start),
- }
- }
- _ => {
- return Err(p
- .error("Expected a block after condition".to_string())
- .into())
- }
- };
-
- let no = match p.peek() {
- Some(TokenType::Else) => {
- _ = p.next();
- match p.peek() {
- Some(TokenType::OpenBrace) => {
- let block_start = p.index;
- // let comments = p.take_comments();
- Some(BlockOrIf::Block(BlockWithRange {
- block: parse_block(p)?,
- range: Range::new(block_start, p.index - block_start),
- }))
- }
- Some(TokenType::If) => {
- // TODO(@paperdave): Save location for the if
- // let range = p.range();
- // let comments = p.take_comments();
- Some(BlockOrIf::If(Box::new(parse_expr_if(p)?)))
- }
- _ => return Err(p.error("Expected a block after else".to_string()).into()),
- }
- }
- _ => None,
- };
-
- Ok(If { condition, yes, no })
-}
diff --git a/crates/parser/parse_fn_declaration.rs b/crates/parser/parse_fn_declaration.rs
deleted file mode 100644
index 50e99a9640c517feced1440f60c7ec494cf0747b..0000000000000000000000000000000000000000
--- a/crates/parser/parse_fn_declaration.rs
+++ /dev/null
@@ -1,127 +0,0 @@
-use ast::{DocComment, Expr, ExprData, FnDecl, Type};
-use log::Range;
-use miette::Result;
-use tokenizer::TokenType;
-
-use crate::parse_block::parse_block;
-use crate::parse_expr::parse_expr;
-use crate::parse_param::parse_param;
-use crate::parse_type::parse_type;
-use crate::Parser;
-
-/// Precondition:
-pub fn parse_fn_declaration(
- p: &mut Parser,
- is_pub: bool,
- comment: Option,
-) -> Result {
- let start = p.index;
- match p.next() {
- Some(TokenType::Fn) => (),
- _ => unreachable!(),
- }
-
- let name = match p.next() {
- Some(TokenType::Identifier(name)) => name,
- _ => return Err(p.error("Expected identifier after 'fn'".to_string()).into()),
- };
-
- match p.peek() {
- Some(TokenType::OpenParen) => p.next(),
- _ => {
- return Err(p
- .error("Expected '(' after function name".to_string())
- .into())
- }
- };
-
- let mut params = Vec::new();
-
- loop {
- match p.peek() {
- Some(TokenType::ClosedParen) => {
- _ = p.next();
- break;
- }
- Some(TokenType::Identifier(_)) => {
- params.push(parse_param(p)?);
-
- match p.peek() {
- Some(TokenType::Comma) => {
- p.next();
- if let Some(TokenType::ClosedParen) = p.peek() {
- p.next();
- break;
- }
- }
- Some(TokenType::ClosedParen) => {
- p.next();
- break;
- }
- _ => {
- let actual = p.peek().map_or("EOF".to_string(), |x| format!("{:?}", x));
- return Err(p
- .error(format!(
- "Expected ',' or ')' after identifier, got {}",
- actual
- ))
- .into());
- }
- };
- }
- _ => {
- return Err(p
- .error("Expected ')' or identifier after '('".to_string())
- .into())
- }
- }
- }
-
- let return_type = match p.peek() {
- Some(TokenType::MinusLt) => {
- _ = p.next();
- parse_type(p)?
- }
- _ => Type::Infer,
- };
-
- let was_in_fn = p.is_in_fn;
- p.is_in_fn = true;
-
- let body = match p.peek() {
- Some(TokenType::OpenBrace) => {
- let block_start = p.index;
- let block = parse_block(p)?;
- Expr::new(
- ExprData::Block(block),
- Range::new(block_start, p.index - block_start),
- None,
- )
- }
- Some(TokenType::EqualLt) => {
- _ = p.next();
- parse_expr(p)?
- }
- _ => {
- let actually = p.peek().map_or("EOF".to_string(), |x| format!("{:?}", x));
- return Err(p
- .error(format!(
- "Expected a block or '=>' to follow function declaration, but got {}",
- actually,
- ))
- .into());
- }
- };
-
- p.is_in_fn = was_in_fn;
-
- Ok(FnDecl {
- name,
- return_type,
- params,
- body,
- is_pub,
- comment,
- range: Range::new(start, p.index - start),
- })
-}
diff --git a/crates/parser/parse_module.rs b/crates/parser/parse_module.rs
deleted file mode 100644
index 725507386a05fa9a56cb15234c7bce005352584b..0000000000000000000000000000000000000000
--- a/crates/parser/parse_module.rs
+++ /dev/null
@@ -1,44 +0,0 @@
-use crate::parse_doc_comment::maybe_read_doc_comment;
-use crate::parse_statement::parse_statement;
-use crate::Parser;
-use ast::Module;
-use miette::Result;
-use tokenizer::TokenType;
-
-pub fn parse_module(p: &mut Parser) -> Result {
- let mut statements = Vec::new();
-
- let file_doc_comment = maybe_read_doc_comment(p);
-
- loop {
- let is_statement_like_expression = p
- .peek()
- .map_or(false, |x| x.is_start_of_statement_like_expression());
-
- let statement = parse_statement(p)?;
-
- match p.peek() {
- Some(TokenType::Semicolon) => {
- while let Some(TokenType::Semicolon) = p.peek() {
- p.next();
- }
- }
- _ => {
- if statement.require_semicolon(is_statement_like_expression) {
- return Err(p
- .error("Expected semicolon after statement".to_string())
- .into());
- }
- }
- };
-
- statements.push(statement);
-
- if p.empty() {
- return Ok(Module {
- statements,
- comment: file_doc_comment,
- });
- }
- }
-}
diff --git a/crates/parser/parse_param.rs b/crates/parser/parse_param.rs
deleted file mode 100644
index d726905cc438ad3064f113cfcdd0cebb165555b6..0000000000000000000000000000000000000000
--- a/crates/parser/parse_param.rs
+++ /dev/null
@@ -1,30 +0,0 @@
-use ast::Param;
-use miette::Result;
-use tokenizer::TokenType;
-
-use crate::{parse_type::parse_type, Parser};
-
-/// Precondition: A token exists and is an identifier
-pub fn parse_param(p: &mut Parser) -> Result {
- let name = match p.next() {
- Some(TokenType::Identifier(name)) => name.clone(),
- _ => unreachable!(),
- };
-
- let param_type = match p.peek() {
- Some(TokenType::Colon) => {
- _ = p.next();
- parse_type(p)?
- }
- _ => {
- return Err(p
- .error("Function parameters require type annotations".to_string())
- .into())
- }
- };
-
- Ok(Param {
- name: name.clone(),
- param_type,
- })
-}
diff --git a/crates/parser/parse_statement.rs b/crates/parser/parse_statement.rs
deleted file mode 100644
index c46427f7a549d2d07593fbc85d2446aae21f4297..0000000000000000000000000000000000000000
--- a/crates/parser/parse_statement.rs
+++ /dev/null
@@ -1,88 +0,0 @@
-use ast::Statement;
-use miette::Result;
-use tokenizer::TokenType;
-
-use crate::parse_declaration::parse_declaration;
-use crate::parse_doc_comment::maybe_read_doc_comment;
-use crate::parse_expr::parse_expr;
-use crate::parse_fn_declaration::parse_fn_declaration;
-use crate::Parser;
-
-pub fn parse_statement(p: &mut Parser) -> Result {
- if let Some(comments) = p.take_comments() {
- return Ok(Statement::Comment(comments));
- }
-
- let m_doc_comment = maybe_read_doc_comment(p);
- if let Some(doc_comment) = m_doc_comment.clone() {
- if p.has_comments() {
- // this implies that comments are immediately after the doc
- return Err(p
- .error_at(format!("Unattached doc comment"), doc_comment.range)
- .into());
- }
- }
-
- match p.peek() {
- Some(TokenType::DocComment(_)) => unreachable!(),
- Some(TokenType::Pub) => {
- if !p.is_top_level {
- return Err(p
- .error("Public declarations are only allowed at the top level".to_string())
- .into());
- }
- _ = p.next();
- match p.peek() {
- Some(TokenType::Var) | Some(TokenType::Let) => Ok(Statement::VarDecl(
- parse_declaration(p, true, m_doc_comment)?,
- )),
- Some(TokenType::Fn) => Ok(Statement::FnDecl(parse_fn_declaration(
- p,
- true,
- m_doc_comment,
- )?)),
- Some(_) => Err(p.error("Expected a declaration".to_string()).into()),
- None => Err(p.error("Unexpected end of file".to_string()).into()),
- }
- }
- Some(TokenType::Var) | Some(TokenType::Let) => Ok(Statement::VarDecl(parse_declaration(
- p,
- false,
- m_doc_comment,
- )?)),
- Some(TokenType::Fn) => Ok(Statement::FnDecl(parse_fn_declaration(
- p,
- false,
- m_doc_comment,
- )?)),
- Some(TokenType::Defer) => {
- let label = p.peek().unwrap();
- Err(p
- .error(format!("'{}' is not yet implemented", label))
- .into())
- }
- Some(TokenType::Struct | TokenType::Enum | TokenType::Trait | TokenType::Impl) => {
- let label = p.peek().unwrap();
- if !p.is_top_level {
- return Err(p
- .error(format!("'{}' is only allowed at the top level", label))
- .into());
- }
-
- Err(p
- .error(format!("'{}' is not yet implemented", label))
- .into())
- }
-
- Some(_) => {
- if let Some(doc_comment) = m_doc_comment {
- Err(p
- .error_at(format!("Unattached doc comment"), doc_comment.range)
- .into())
- } else {
- Ok(Statement::Expr(parse_expr(p)?))
- }
- }
- None => Err(p.error("Unexpected end of file".to_string()).into()),
- }
-}
diff --git a/crates/parser/parse_type.rs b/crates/parser/parse_type.rs
deleted file mode 100644
index c60afcedf0c7494c203089fe4f8c4a4e29a80841..0000000000000000000000000000000000000000
--- a/crates/parser/parse_type.rs
+++ /dev/null
@@ -1,26 +0,0 @@
-use ast::Type;
-use miette::Result;
-
-use tokenizer::TokenType;
-
-use crate::Parser;
-
-pub fn parse_type(p: &mut Parser) -> Result {
- match p.next() {
- Some(TokenType::Identifier(id)) => match id.as_str() {
- "int" => Ok(Type::Int),
- "float" => Ok(Type::Float),
- "bigint" => Ok(Type::BigInt),
- "bool" => Ok(Type::Bool),
- "string" => Ok(Type::String),
- invalid_type => Err(p
- .error(format!(
- "Invalid type '{}' (TODO: add custom type support)",
- invalid_type
- ))
- .into()),
- },
- Some(unexpected) => Err(p.error(format!("Expected type, got {}", unexpected)).into()),
- None => Err(p.error("Expected type, got EOF".to_string()).into()),
- }
-}
diff --git a/crates/parser/parser.rs b/crates/parser/parser.rs
deleted file mode 100644
index 3d601ffe6451edfc46a0503906356360ee113047..0000000000000000000000000000000000000000
--- a/crates/parser/parser.rs
+++ /dev/null
@@ -1,150 +0,0 @@
-mod parse_block;
-mod parse_call_arguments;
-mod parse_declaration;
-mod parse_doc_comment;
-mod parse_expr;
-mod parse_expr_basic;
-mod parse_expr_if;
-mod parse_fn_declaration;
-mod parse_module;
-mod parse_param;
-mod parse_statement;
-mod parse_type;
-
-use ast::Comment;
-use ast::Module;
-use log::Range;
-use miette::Diagnostic;
-use miette::Result;
-use parse_module::parse_module;
-use thiserror::Error;
-use tokenizer::tokenize;
-use tokenizer::Token;
-use tokenizer::TokenType;
-
-pub fn parse(tokens: Vec) -> Result {
- let mut p = Parser::new(tokens);
- parse_module(&mut p)
-}
-
-pub fn parse_from_string(string: String) -> Result {
- let tokens = tokenize(string)?;
- parse(tokens)
-}
-
-#[derive(Error, Diagnostic, Debug)]
-#[error("ParseError")]
-pub struct ParseError {
- #[help]
- error: String,
- range: Option,
-}
-
-impl ParseError {
- pub fn new(error: String, range: Option) -> ParseError {
- ParseError { error, range }
- }
-}
-
-#[derive(Debug)]
-pub struct Parser {
- /// TODO(@paperdave): switch to streaming tokenizer + one that can
- /// report line and column info
- tokens: Vec,
- cursor: usize,
-
- /// Current index in the string
- pub index: usize,
-
- /// Whether or not the parser is at the top level.
- /// Currently controls the validity of `pub` keywords.
- pub is_top_level: bool,
- /// Whether or not the parser is inside a function.
- /// Controls the validity of the `return` keyword.
- pub is_in_fn: bool,
-
- comments: Vec,
-}
-
-impl Parser {
- pub fn new(tokens: Vec) -> Parser {
- Parser {
- tokens,
- cursor: 0,
- index: 0,
- is_top_level: true,
- is_in_fn: false,
- comments: vec![],
- }
- }
-
- pub fn empty(&self) -> bool {
- self.cursor == self.tokens.len()
- }
-
- pub fn next(&mut self) -> Option {
- if self.cursor == self.tokens.len() {
- return None;
- }
-
- loop {
- let token = self.tokens[self.cursor].clone();
- self.index = token.range.start;
- match token.variant {
- TokenType::Comment(comment) => {
- self.comments.push(comment);
- self.cursor += 1;
- continue;
- }
- token => {
- self.cursor += 1;
- return Some(token);
- }
- }
- }
- }
-
- pub fn take_comments(&mut self) -> Option {
- if self.comments.len() == 0 {
- return None;
- }
- let comments = self.comments.clone();
- self.comments = vec![];
- Some(Comment {
- data: comments.join("\n"),
- })
- }
-
- pub fn has_comments(&mut self) -> bool {
- self.peek();
- self.comments.len() > 0
- }
-
- pub fn peek(&mut self) -> Option {
- loop {
- if self.cursor == self.tokens.len() {
- return None;
- }
-
- match self.tokens[self.cursor].clone().variant {
- TokenType::Comment(comment) => {
- self.comments.push(comment);
- self.cursor += 1;
- continue;
- }
- token => {
- return Some(token);
- }
- }
- }
- }
-
- pub fn error(&self, error: String) -> ParseError {
- // TODO(@lino-levan): Add location information
- ParseError::new(error, None)
- }
-
- fn error_at(&self, error: String, range: Range) -> ParseError {
- ParseError::new(error, Some(range))
- }
-}
diff --git a/crates/sema/Cargo.toml b/crates/sema/Cargo.toml
deleted file mode 100644
index d9a6f7ea10286ec4418afd61c1538671280ce8b4..0000000000000000000000000000000000000000
--- a/crates/sema/Cargo.toml
+++ /dev/null
@@ -1,14 +0,0 @@
-[package]
-name = "sema"
-version = "0.1.0"
-edition = "2021"
-
-[lib]
-path = "./sema.rs"
-
-[dependencies]
-miette.workspace = true
-thiserror.workspace = true
-num.workspace = true
-
-ast = { path = "../ast" }
\ No newline at end of file
diff --git a/crates/sema/ast_type_coersion.rs b/crates/sema/ast_type_coersion.rs
deleted file mode 100644
index 2fab10b08f06b009feda2523d100641b89dc0d52..0000000000000000000000000000000000000000
--- a/crates/sema/ast_type_coersion.rs
+++ /dev/null
@@ -1,54 +0,0 @@
-// use ast::{BinaryOperator, ExprData, Type};
-
-impl TypeCoerceable for ExprData {
- fn to_type(&self) -> Option {
- match self {
- ExprData::Block(block) => {
- if (block.has_implicit_return) {
- block.statements.last().unwrap().to_type()
- } else {
- Some(Type::Void)
- }
- }
- ExprData::Binary { op, left, right } => {
- // TODO: this likely has to do further type checking
- let left_type = left.to_type();
- match op {
- BinaryOperator::Add => left_type,
- BinaryOperator::Sub => left_type,
- BinaryOperator::Mul => left_type,
- BinaryOperator::Div => left_type,
- BinaryOperator::Pow => left_type,
- BinaryOperator::Eq => Some(Type::Bool),
- BinaryOperator::Gt => Some(Type::Bool),
- BinaryOperator::Lt => Some(Type::Bool),
- BinaryOperator::Gte => Some(Type::Bool),
- BinaryOperator::Lte => Some(Type::Bool),
- BinaryOperator::Lshift => Some(Type::Int),
- BinaryOperator::Rshift => Some(Type::Int),
- BinaryOperator::BitAnd => Some(Type::Int),
- BinaryOperator::BitOr => Some(Type::Int),
- BinaryOperator::BitXor => Some(Type::Int),
- BinaryOperator::Mod => Some(Type::Int),
- BinaryOperator::NotEq => Some(Type::Int),
-
- BinaryOperator::And => Some(Type::Bool),
- BinaryOperator::Or => Some(Type::Bool),
- }
- }
- ExprData::Unary { op, value } => todo!(),
- ExprData::Function {
- name,
- params,
- return_type,
- block,
- } => Some(return_type),
- ExprData::Call { base, arguments } => None, // oh shit. yeah this trait isn't implementable
- ExprData::Builtin { id, arguments } => todo!(),
- ExprData::If(_) => todo!(),
- ExprData::Return(_) => todo!(),
- ExprData::Literal(_) => todo!(),
- ExprData::Identifier(_) => todo!(),
- }
- }
-}
diff --git a/crates/sema/sema.rs b/crates/sema/sema.rs
deleted file mode 100644
index 899c5d9433436f4bf7f4f85d55a96d821cc88572..0000000000000000000000000000000000000000
--- a/crates/sema/sema.rs
+++ /dev/null
@@ -1,98 +0,0 @@
-use ast::Module;
-use miette::Result;
-
-pub fn analyze_module(_block: &mut Module) -> Result<()> {
- //
- Ok(())
-}
-
-// for statement in &block.statements {
-// check_expression(statement)?;
-// }
-
-// Ok(())
-// }
-
-// pub fn resolve_type(expr: &Ast) -> Result {
-// match expr {
-// Ast::Number(value) => Ok(Type::NumberLiteral(value.clone())),
-// Ast::Declaration {
-// name,
-// decl_type,
-// value,
-// is_constant,
-// } => Ok(Type::NoReturn),
-// Ast::Binary {
-// operator,
-// left,
-// right,
-// } => merge_types(resolve_type(left)?, resolve_type(right)?),
-// Ast::Function {
-// name,
-// arguments,
-// return_type,
-// block,
-// } => todo!(),
-// Ast::FunctionCall { name, arguments } => todo!(),
-// Ast::If {
-// condition,
-// value,
-// else_value,
-// } => todo!(),
-// Ast::Return(_) => Ok(Type::NoReturn),
-// Ast::Public(_) => todo!(), // TODO: remove Public
-// Ast::Block(_) => todo!(),
-// Ast::Bool(_) => Ok(Type::Bool),
-// Ast::String(_) => Ok(Type::String),
-// Ast::Identifier(_) => todo!(),
-// Ast::BuiltIn(_) => todo!(),
-
-// Ast::Comment(_) => unreachable!(),
-// Ast::DocComment(_) => unreachable!(),
-// }
-// }
-
-// pub fn check_expression(expr: &Ast) -> Result<()> {
-// panic!()
-// match expr {
-// Ast::Declaration {
-// name,
-// decl_type: return_type,
-// value,
-// is_constant,
-// } => {
-// check_expression(value)?;
-// // coerce_type(return_type.clone(), resolve_type(value)?)?;
-// }
-// _ => {}
-// // Ast::Operation {
-// // operator,
-// // left,
-// // right,
-// // } => Ok(()),
-// // Ast::Function {
-// // name,
-// // arguments,
-// // return_type,
-// // block,
-// // } => Ok(()),
-// // Ast::FunctionCall { name, arguments } => Ok(()),
-// // Ast::If {
-// // condition,
-// // value,
-// // else_value,
-// // } => Ok(()),
-// // Ast::Return(_) => Ok(()),
-// // Ast::Public(_) => Ok(()),
-// // Ast::Block(_) => Ok(()),
-// // Ast::Number(_) => Ok(()),
-// // Ast::Bool(_) => Ok(()),
-// // Ast::String(_) => Ok(()),
-// // Ast::Identifier(_) => Ok(()),
-// // Ast::BuiltIn(_) => Ok(()),
-// // Ast::Comment(_) => Ok(()),
-// // Ast::DocComment(_) => Ok(()),
-// }
-
-// Ok(())
-// }
diff --git a/crates/sema/sema_scope.rs b/crates/sema/sema_scope.rs
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/crates/tokenizer/Cargo.toml b/crates/tokenizer/Cargo.toml
deleted file mode 100644
index 5a51640ba63101310df8787cb92377295a1b0024..0000000000000000000000000000000000000000
--- a/crates/tokenizer/Cargo.toml
+++ /dev/null
@@ -1,13 +0,0 @@
-[package]
-name = "tokenizer"
-version = "0.1.0"
-edition = "2021"
-
-[lib]
-path="./tokenizer.rs"
-
-[dependencies]
-miette.workspace = true
-thiserror.workspace = true
-
-log = { path = "../log" }
diff --git a/crates/tokenizer/tokenizer.rs b/crates/tokenizer/tokenizer.rs
deleted file mode 100644
index 8fdecab4fa033aaab86efd1aa178031f99011147..0000000000000000000000000000000000000000
--- a/crates/tokenizer/tokenizer.rs
+++ /dev/null
@@ -1,443 +0,0 @@
-use log::Range;
-use miette::{Diagnostic, Result};
-use thiserror::Error;
-
-#[derive(Error, Diagnostic, Debug)]
-#[error("UnknownCharacter")]
-struct UnknownCharacter {
- #[help]
- error: String,
-}
-
-#[derive(Debug, Clone, PartialEq)]
-pub enum TokenType {
- // Literals
- Identifier(String),
- Number(f64),
- String(String),
- // Comments
- Comment(String),
- DocComment(String),
- // Keywords
- Defer,
- Else,
- Enum,
- False,
- Fn,
- If,
- Impl,
- Let,
- Pub,
- Return,
- Struct,
- Trait,
- True,
- Var,
- Void,
- // Symbols
- And, // &
- AndAnd, // &&
- Bang, // !
- BangEqual, // !=
- EqualLt, // =>
- Caret, // ^
- ClosedBrace, // }
- ClosedBracket, // ]
- ClosedParen, // )
- Colon, // :
- Comma, // ,
- Dot, // .
- Equal, // =
- EqualEqual, // ==
- Gt, // >
- GtEqual, // >=
- GtGt, // >>
- GtGtGt, // >>>
- Lt, // <
- LtEqual, // <=
- LtLt, // <<
- Minus, // -
- OpenBrace, // {
- OpenBracket, // [
- OpenParen, // (
- Or, // |
- OrOr, // ||
- Percent, // %
- Plus, // +
- MinusLt, // ->
- Semicolon, // ;
- Slash, // /
- Star, // *
- StarStar, // **
- Tilde, // ~
-}
-
-impl TokenType {
- pub fn is_start_of_statement_like_expression(&self) -> bool {
- match &self {
- TokenType::If | TokenType::OpenBrace => true,
- _ => false,
- }
- }
-}
-
-impl std::fmt::Display for TokenType {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- let token = match &self {
- TokenType::Identifier(_) => "identifier".to_string(),
-
- TokenType::Number(n) => n.to_string(),
- TokenType::String(s) => format!("\"{}\"", s),
- TokenType::Comment(_) => "comment".to_string(),
- TokenType::DocComment(_) => "documentation comment".to_string(),
-
- TokenType::Defer => "defer".to_string(),
- TokenType::Else => "else".to_string(),
- TokenType::Enum => "enum".to_string(),
- TokenType::False => "false".to_string(),
- TokenType::Fn => "fn".to_string(),
- TokenType::If => "if".to_string(),
- TokenType::Impl => "impl".to_string(),
- TokenType::Let => "let".to_string(),
- TokenType::Pub => "pub".to_string(),
- TokenType::Return => "return".to_string(),
- TokenType::Struct => "struct".to_string(),
- TokenType::Trait => "trait".to_string(),
- TokenType::True => "true".to_string(),
- TokenType::Var => "var".to_string(),
- TokenType::Void => "void".to_string(),
-
- TokenType::And => "&".to_string(),
- TokenType::AndAnd => "&&".to_string(),
- TokenType::Bang => "!".to_string(),
- TokenType::BangEqual => "!=".to_string(),
- TokenType::EqualLt => "=>".to_string(),
- TokenType::Caret => "^".to_string(),
- TokenType::ClosedBrace => "}".to_string(),
- TokenType::ClosedBracket => "]".to_string(),
- TokenType::ClosedParen => ")".to_string(),
- TokenType::Colon => ":".to_string(),
- TokenType::Comma => ",".to_string(),
- TokenType::Dot => ".".to_string(),
- TokenType::Equal => "=".to_string(),
- TokenType::EqualEqual => "==".to_string(),
- TokenType::Gt => ">".to_string(),
- TokenType::GtEqual => ">=".to_string(),
- TokenType::GtGt => ">>".to_string(),
- TokenType::GtGtGt => ">>>".to_string(),
- TokenType::Lt => "<".to_string(),
- TokenType::LtEqual => "<=".to_string(),
- TokenType::LtLt => "<<".to_string(),
- TokenType::Minus => "-".to_string(),
- TokenType::OpenBrace => "{".to_string(),
- TokenType::OpenBracket => "[".to_string(),
- TokenType::OpenParen => "(".to_string(),
- TokenType::Or => "|".to_string(),
- TokenType::OrOr => "||".to_string(),
- TokenType::Percent => "%".to_string(),
- TokenType::Plus => "+".to_string(),
- TokenType::MinusLt => "->".to_string(),
- TokenType::Semicolon => ";".to_string(),
- TokenType::Slash => "/".to_string(),
- TokenType::Star => "*".to_string(),
- TokenType::StarStar => "**".to_string(),
- TokenType::Tilde => "~".to_string(),
- };
-
- write!(f, "{}", token)
- }
-}
-
-#[derive(Debug, Clone)]
-pub struct Token {
- pub variant: TokenType,
- pub range: Range,
-}
-
-impl Token {
- pub fn new(variant: TokenType, start: usize, length: usize) -> Self {
- Self {
- variant,
- range: Range::new(start, length),
- }
- }
-}
-
-// TODO(@paperdave): rewrite this to not use a Vec. we probably dont need it. also store Range
-pub fn tokenize(content: String) -> Result> {
- let mut tokens: Vec = Vec::new();
- let mut chars = content.chars().enumerate().peekable();
-
- loop {
- match chars.next() {
- Some((start, c)) => {
- let token = match c {
- 'a'..='z' | 'A'..='Z' | '_' | '@' => {
- let mut current_token = String::new();
- current_token.push(c);
- loop {
- let c = chars.peek();
- if c.is_none() {
- break;
- }
- let (_, c) = c.unwrap();
- match c {
- 'a'..='z' | 'A'..='Z' | '_' | '0'..='9' => {
- current_token.push(*c);
- chars.next();
- }
- _ => break,
- }
- }
-
- let length = current_token.len();
-
- let variant = match current_token.as_str() {
- "defer" => TokenType::Defer,
- "else" => TokenType::Else,
- "enum" => TokenType::Enum,
- "false" => TokenType::False,
- "fn" => TokenType::Fn,
- "if" => TokenType::If,
- "impl" => TokenType::Impl,
- "let" => TokenType::Let,
- "pub" => TokenType::Pub,
- "return" => TokenType::Return,
- "struct" => TokenType::Struct,
- "trait" => TokenType::Trait,
- "true" => TokenType::True,
- "var" => TokenType::Var,
- "void" => TokenType::Void,
- _ => TokenType::Identifier(current_token),
- };
-
- Token::new(variant, start, length)
- }
- '"' => {
- let mut length = 1;
- let mut current_token = String::new();
- loop {
- let c = chars.next();
- if c.is_none() {
- break;
- }
- let (_, c) = c.unwrap();
- length += 1;
- match c {
- '"' => break,
- _ => current_token.push(c),
- }
- }
- Token::new(TokenType::String(current_token), start, length)
- }
- '0'..='9' => {
- let mut current_token = String::new();
- current_token.push(c);
- loop {
- let c = chars.peek();
- if c.is_none() {
- break;
- }
- let (_, c) = c.unwrap();
- match c {
- '0'..='9' => {
- current_token.push(*c);
- chars.next();
- }
- _ => break,
- }
- }
- match current_token.parse::() {
- Ok(v) => Token::new(TokenType::Number(v), start, current_token.len()),
- Err(_) => {
- return Err(UnknownCharacter {
- error: format!("Unknown character: {}", current_token),
- }
- .into())
- }
- }
- }
- ':' => Token::new(TokenType::Colon, start, 1),
- '=' => match chars.peek() {
- Some((_, '=')) => {
- chars.next();
- Token::new(TokenType::EqualEqual, start, 2)
- }
- Some((_, '>')) => {
- chars.next();
- Token::new(TokenType::EqualLt, start, 2)
- }
- _ => Token::new(TokenType::Equal, start, 1),
- },
- ';' => Token::new(TokenType::Semicolon, start, 1),
- '>' => match chars.peek() {
- Some((_, '=')) => {
- chars.next();
- Token::new(TokenType::GtEqual, start, 2)
- }
- Some((_, '>')) => {
- chars.next();
- match chars.peek() {
- Some((_, '>')) => {
- chars.next();
- Token::new(TokenType::GtGtGt, start, 3)
- }
- _ => Token::new(TokenType::GtGt, start, 2),
- }
- }
- _ => Token::new(TokenType::Gt, start, 1),
- },
- '<' => match chars.peek() {
- Some((_, '=')) => {
- chars.next();
- Token::new(TokenType::LtEqual, start, 2)
- }
- Some((_, '<')) => {
- chars.next();
- Token::new(TokenType::LtLt, start, 2)
- }
- _ => Token::new(TokenType::Lt, start, 1),
- },
- '+' => Token::new(TokenType::Plus, start, 1),
- '-' => match chars.peek() {
- Some((_, '>')) => {
- chars.next();
- Token::new(TokenType::MinusLt, start, 2)
- }
- _ => Token::new(TokenType::Minus, start, 1),
- },
- '*' => match chars.peek() {
- Some((_, '*')) => {
- chars.next();
- Token::new(TokenType::StarStar, start, 2)
- }
- _ => Token::new(TokenType::Star, start, 1),
- },
- '/' => {
- match chars.peek() {
- Some((_, '/')) => {
- chars.next(); // Consume the second slash
- let mut length = 2;
-
- let is_doc_comment = match chars.peek() {
- Some((_, '/')) => {
- chars.next(); // Consume the third slash
- length += 1;
- true
- }
- _ => false,
- };
-
- let mut comment = String::new();
- loop {
- // skip leading whitespace
- while let Some((_, c)) = chars.peek() {
- if !c.is_whitespace() {
- break;
- }
- chars.next();
- length += 1;
- }
- // read the comment
- while let Some((_, c)) = chars.next() {
- length += 1;
- if c == '\n' || c == '\r' {
- break;
- }
- comment.push(c);
- }
- // trim trailing whitespace on line
- comment = comment.trim_end().to_string();
- // skip to next non-whitespace character
- while let Some((_, c)) = chars.peek() {
- if !c.is_whitespace() {
- break;
- }
- chars.next();
- length += 1;
- }
- // check if we've hit another comment
- let mut tmp = chars.clone();
-
- let is_another_comment = match tmp.peek() {
- Some((_, '/')) => {
- tmp.next(); // Consume the second slash
- match tmp.peek() {
- Some((_, '/')) => {
- tmp.next(); // Consume the third slash
- is_doc_comment
- }
- _ => !is_doc_comment,
- }
- }
- _ => false,
- };
-
- if is_another_comment {
- // apply our temporary iterator to the main iterator
- comment.push('\n');
- chars = tmp;
- chars.next();
- length += 1;
- } else {
- break;
- }
- }
-
- if is_doc_comment {
- Token::new(TokenType::DocComment(comment), start, length)
- } else {
- Token::new(TokenType::Comment(comment), start, length)
- }
- }
- _ => Token::new(TokenType::Slash, start, 1),
- }
- }
- ',' => Token::new(TokenType::Comma, start, 1),
- '(' => Token::new(TokenType::OpenParen, start, 1),
- ')' => Token::new(TokenType::ClosedParen, start, 1),
- '{' => Token::new(TokenType::OpenBrace, start, 1),
- '}' => Token::new(TokenType::ClosedBrace, start, 1),
- '[' => Token::new(TokenType::OpenBracket, start, 1),
- ']' => Token::new(TokenType::ClosedBracket, start, 1),
- '&' => match chars.peek() {
- Some((_, '&')) => {
- chars.next();
- Token::new(TokenType::AndAnd, start, 2)
- }
- _ => Token::new(TokenType::And, start, 1),
- },
- '|' => match chars.peek() {
- Some((_, '|')) => {
- chars.next();
- Token::new(TokenType::OrOr, start, 2)
- }
- _ => Token::new(TokenType::Or, start, 1),
- },
- '!' => match chars.peek() {
- Some((_, '=')) => {
- chars.next();
- Token::new(TokenType::BangEqual, start, 2)
- }
- _ => Token::new(TokenType::Bang, start, 1),
- },
- '^' => Token::new(TokenType::Caret, start, 1),
- '%' => Token::new(TokenType::Percent, start, 1),
- '~' => Token::new(TokenType::Tilde, start, 1),
- '.' => Token::new(TokenType::Dot, start, 1),
- s if s.is_whitespace() => continue,
- _ => {
- return Err(UnknownCharacter {
- error: format!("Unexpected '{}' while tokenizing", c),
- }
- .into())
- }
- };
- tokens.push(token);
- }
- None => break,
- }
- }
-
- Ok(tokens)
-}
diff --git a/examples/basics/associativity.blight b/examples/basics/associativity.blight
deleted file mode 100644
index 5bc76df04a89298c5be31f3f45f6b38c383ac986..0000000000000000000000000000000000000000
--- a/examples/basics/associativity.blight
+++ /dev/null
@@ -1,6 +0,0 @@
-let print = @raw("console.log");
-print(1 * 2 * 3);
-print(1 * 2 / 3);
-print(1 + 2 ** 3);
-print(1 ** 2 ** 3);
-print(1 * 2 + 3);
diff --git a/examples/basics/comments.blight b/examples/basics/comments.blight
deleted file mode 100644
index cef4db9adcbbb9a7735efe8c97e0bea4e4a39b12..0000000000000000000000000000000000000000
--- a/examples/basics/comments.blight
+++ /dev/null
@@ -1,9 +0,0 @@
-// this is a regular comment
-let x: int = 0;
-
-/// this is a doc comment
-let y: int = 0;
-
-/// Doc comments can be multiple lines.
-/// This can be done by simply adding another triple-slash to the next line.
-let z: int = 0;
diff --git a/examples/basics/control.blight b/examples/basics/control.blight
deleted file mode 100644
index f6e3bdb1db079dc14869607c8fc2821eee7a89f2..0000000000000000000000000000000000000000
--- a/examples/basics/control.blight
+++ /dev/null
@@ -1,17 +0,0 @@
-let print = @raw("console.log");
-
-let x: bool = true;
-
-if x {
- print("x is true!");
-}
-
-let y: int = 3;
-
-if y == 1 {
- print("y is 1!");
-} else if y == 2 {
- print("y is 2!");
-} else {
- print("y is not 1 or 2!");
-}
diff --git a/examples/basics/defer.blight b/examples/basics/defer.blight
deleted file mode 100644
index c24839ea1ad2b81c5bbd49899ff13922ac5c4bd7..0000000000000000000000000000000000000000
--- a/examples/basics/defer.blight
+++ /dev/null
@@ -1,12 +0,0 @@
-let print = @raw("console.log");
-
-{
- print("1");
- defer print("6");
- {
- print("2");
- defer print("4");
- print("3");
- }
- print("5");
-}
diff --git a/examples/basics/expressions.blight b/examples/basics/expressions.blight
deleted file mode 100644
index 4897a92481c947df2cefd5171363c5d919241e9f..0000000000000000000000000000000000000000
--- a/examples/basics/expressions.blight
+++ /dev/null
@@ -1,11 +0,0 @@
-// everything is an expression in blight
-// everything.
-// hehe
-
-let x: int = 5 + {
- let y: int = 5;
- y
-};
-
-let print = @raw("console.log");
-print(x); // 10
diff --git a/examples/basics/fib.blight b/examples/basics/fib.blight
deleted file mode 100644
index bfa4b5623183aa460e1f0b79e0988f660ab4def4..0000000000000000000000000000000000000000
--- a/examples/basics/fib.blight
+++ /dev/null
@@ -1,12 +0,0 @@
-pub fn fib(n: int) -> int {
- if n == 0 {
- 0
- } else if n == 1 {
- 1
- } else {
- fib(n - 1) + fib(n - 2)
- }
-}
-
-let print = @raw("console.log");
-print(fib(10));
diff --git a/examples/basics/functions.blight b/examples/basics/functions.blight
deleted file mode 100644
index 37ca9d3a2dce9143b9e7b2dbd50819abff151942..0000000000000000000000000000000000000000
--- a/examples/basics/functions.blight
+++ /dev/null
@@ -1,8 +0,0 @@
-fn bingus(x: int, y: int) -> int {
- x * y
-}
-
-let result: int = bingus(10, 25);
-
-let print = @raw("console.log");
-print(result);
diff --git a/examples/basics/operations.blight b/examples/basics/operations.blight
deleted file mode 100644
index 55dc5693ebb9d507fd598284e1510811f34cbc03..0000000000000000000000000000000000000000
--- a/examples/basics/operations.blight
+++ /dev/null
@@ -1,4 +0,0 @@
-let a: int = 10 - 2 / 4 * 15;
-
-let print = @raw("console.log");
-print(a);
diff --git a/examples/basics/shadowing.blight b/examples/basics/shadowing.blight
deleted file mode 100644
index 3e96605fa4d98e020ae275232d4cc06ed958c9c0..0000000000000000000000000000000000000000
--- a/examples/basics/shadowing.blight
+++ /dev/null
@@ -1,14 +0,0 @@
-let print = @raw("console.log");
-
-let x: int = 0;
-let x: int = 1;
-print(x); // 1
-
-let y: int = 0;
-{
- let y: int = 1;
- {
- let y: int = 2;
- }
-}
-print(y); // 0
diff --git a/examples/basics/stupid.blight b/examples/basics/stupid.blight
deleted file mode 100644
index 4144819ae054ff63f524006f9d3c00e2e4793ba8..0000000000000000000000000000000000000000
--- a/examples/basics/stupid.blight
+++ /dev/null
@@ -1,2 +0,0 @@
-
-fn add(x: int, y: int) => x + y
diff --git a/examples/basics/variables.blight b/examples/basics/variables.blight
deleted file mode 100644
index 302be86a6474de450a084ae10406e333f2afd1b5..0000000000000000000000000000000000000000
--- a/examples/basics/variables.blight
+++ /dev/null
@@ -1,10 +0,0 @@
-var x: int = 0;
-var y: float = 0;
-var z: bigint = 0;
-
-let a: int = 0;
-let b: float = 0;
-let c: bigint = 0;
-
-let print = @raw("console.log");
-print(x);
diff --git a/examples/basics/weird.blight b/examples/basics/weird.blight
deleted file mode 100644
index 80b5ea000738ae9763f383f35a18b2a665448958..0000000000000000000000000000000000000000
--- a/examples/basics/weird.blight
+++ /dev/null
@@ -1,14 +0,0 @@
-// Just trying to break the blight compiler
-fn bingus(y: bool) -> int {
- let x: int = 5 + {
- if y {
- return 100000;
- }
- 6
- };
- x
-}
-
-let print = @raw("console.log");
-print(bingus(true));
-print(bingus(false));
diff --git a/examples/interop/blight_in_js/add.blight b/examples/interop/blight_in_js/add.blight
deleted file mode 100644
index dc247a01e44bd475fe7276c0714c64617ef1a282..0000000000000000000000000000000000000000
--- a/examples/interop/blight_in_js/add.blight
+++ /dev/null
@@ -1,5 +0,0 @@
-pub fn add(x: int, y: int) -> int {
- x + y
-}
-
-pub let TOP_LEVEL: int = 100;
diff --git a/examples/interop/blight_in_js/main.js b/examples/interop/blight_in_js/main.js
deleted file mode 100644
index 18c652b5967116f7639c152c7fbebd1760631755..0000000000000000000000000000000000000000
--- a/examples/interop/blight_in_js/main.js
+++ /dev/null
@@ -1,4 +0,0 @@
-import { add, TOP_LEVEL } from "./add.js";
-
-console.log(add(1, 2));
-console.log(TOP_LEVEL);
diff --git a/examples/interop/deno.blight b/examples/interop/deno.blight
deleted file mode 100644
index 1b5fd3af6cad1c63f41f5f41f55c17d25bc98de0..0000000000000000000000000000000000000000
--- a/examples/interop/deno.blight
+++ /dev/null
@@ -1,7 +0,0 @@
-let writeTextFile = @raw("Deno.writeTextFile");
-
-fn x() -> string {
- "hello"
-}
-
-writeTextFile("hello.txt", x());
diff --git a/src/compiler/Air.Dump.zig b/src/compiler/Air.Dump.zig
new file mode 100644
index 0000000000000000000000000000000000000000..f2a3837eddd4782a15e7d862430590917ee9b3d3
--- /dev/null
+++ b/src/compiler/Air.Dump.zig
@@ -0,0 +1,299 @@
+//! This implements a debug formatter for Teg
+air: Air,
+ip: *const InternPool,
+
+const State = struct {
+ arena: std.mem.Allocator,
+ air: *const Air,
+ ip: *const InternPool,
+ indent: usize = 0,
+ tty: std.io.tty.Config,
+ visited: std.AutoHashMapUnmanaged(Inst.Index, void) = .{},
+
+ fn newline(state: State, writer: anytype) !void {
+ try writer.writeByte('\n');
+ if (state.indent > 0)
+ try writer.writeByteNTimes(' ', state.indent * 2);
+ }
+
+ fn dumpInst(s: *State, w: anytype, inst: Inst.Index) @TypeOf(w).Error!void {
+ const air = s.air;
+ const tty = s.tty;
+ const tag = s.air.instTag(inst);
+
+ try tty.setColor(w, switch (s.air.instFlags(inst).uses) {
+ .none => .bright_black,
+ .multi => .red,
+ .single => .magenta,
+ });
+ try w.writeByte(switch (s.air.instFlags(inst).uses) {
+ .none => 'X',
+ .multi => 'M',
+ .single => 'S',
+ });
+ try tty.setColor(w, .cyan);
+
+ try w.print(" #{} = ", .{
+ @intFromEnum(inst),
+ });
+
+ try tty.setColor(w, .cyan);
+ try w.print("{s}", .{@tagName(tag)});
+ try tty.setColor(w, .reset);
+
+ if ((try s.visited.fetchPut(s.arena, inst, {})) != null) {
+ try tty.setColor(w, .red);
+ try w.writeAll(" [error: recursive tree]");
+ try tty.setColor(w, .reset);
+ return;
+ }
+
+ switch (tag) {
+ .block => {
+ const index_list: Air.IndexList = air.instData(.block, inst);
+ const insts = index_list.slice(air);
+ try s.dumpInstList(w, insts, '{', '}');
+ },
+ .js_global_ref => {
+ const name: InternPool.String.WithLen = air.instData(.js_global_ref, inst);
+ try tty.setColor(w, .magenta);
+ try w.print(" {s}", .{name.toSlice(s.ip)});
+ try tty.setColor(w, .reset);
+ },
+ .arg => {
+ const arg: Air.Arg = air.instData(.arg, inst);
+ try w.writeAll(" ");
+ try tty.setColor(w, .yellow);
+ try w.print("{d}", .{arg.arg_number});
+ try tty.setColor(w, .reset);
+ try w.writeAll(": ");
+ try s.dumpValue(w, arg.type);
+ },
+ .@"break" => {
+ const brk: Air.Break = air.instData(.@"break", inst);
+ try w.writeAll("(");
+ try s.dumpInstRef(w, brk.block_ref);
+ try w.writeAll(", ");
+ try s.dumpRef(w, brk.value);
+ try w.writeAll(")");
+ },
+ inline .add,
+ .mul,
+ .sub,
+ .div,
+ .lt,
+ .gt,
+ .eq,
+ => |comptime_tag| {
+ const bin: Air.Bin = air.instData(comptime_tag, inst);
+ try w.writeAll("(");
+ try s.dumpRef(w, bin.lhs);
+ try w.writeAll(", ");
+ try s.dumpRef(w, bin.rhs);
+ try w.writeAll(")");
+ },
+ inline .negate => |comptime_tag| {
+ const bin: Air.Un = air.instData(comptime_tag, inst);
+ try w.writeAll("(");
+ try s.dumpRef(w, bin.operand);
+ try w.writeAll(")");
+ },
+ .field => {
+ const field: Air.Field = air.instData(.field, inst);
+ try w.writeAll("(");
+ try s.dumpRef(w, field.base);
+ try w.writeAll(", ");
+ try tty.setColor(w, .magenta);
+ try w.print("{s}", .{field.field.toSlice(s.ip)});
+ try tty.setColor(w, .reset);
+ try w.writeAll(")");
+ },
+ .call => {
+ const call: Air.Call = air.instData(.call, inst);
+ const call_data = call.call_data.get(air);
+ try w.writeAll("(");
+ try s.dumpRef(w, call.base);
+ try w.writeAll(", ");
+ try s.dumpRefList(w, call_data.args.slice(air), '[', ']');
+ try w.writeAll(")");
+ },
+ .@"if" => {
+ const if_inst: Air.If = air.instData(.@"if", inst);
+ const if_data: Air.If.Data = if_inst.data.get(air);
+ try w.writeAll("(");
+ try s.dumpRef(w, if_inst.cond);
+ try w.writeAll(", {");
+ s.indent += 1;
+ try s.newline(w);
+ try w.writeAll("yes: ");
+ try s.dumpRefRecurse(w, if_data.yes);
+ try s.newline(w);
+ try w.writeAll("no: ");
+ try s.dumpRefRecurse(w, if_data.no);
+ s.indent -= 1;
+ try s.newline(w);
+ try w.writeAll("}");
+ },
+ }
+ }
+
+ fn dumpRef(s: *State, w: anytype, ref: Inst.Ref) !void {
+ switch (ref.unwrap()) {
+ .inst => |i| try s.dumpInstRef(w, i),
+ .value => |v| try s.dumpValue(w, v),
+ }
+ }
+
+ fn dumpRefRecurse(s: *State, w: anytype, ref: Inst.Ref) !void {
+ switch (ref.unwrap()) {
+ .inst => |i| try s.dumpInst(w, i),
+ .value => |v| try s.dumpValue(w, v),
+ }
+ }
+
+ fn dumpInstRef(s: *State, w: anytype, inst: Inst.Index) @TypeOf(w).Error!void {
+ try s.tty.setColor(w, .red);
+ try w.print("#{d}", .{@intFromEnum(inst)});
+ try s.tty.setColor(w, .reset);
+ }
+
+ fn dumpValue(s: *State, w: anytype, val: Value) !void {
+ const tty = s.tty;
+ switch (val.toKey(s.ip)) {
+ .simple_type => |t| {
+ try tty.setColor(w, .blue);
+ try w.print("{s}", .{@tagName(t)});
+ try tty.setColor(w, .reset);
+ },
+ .simple_value => |t| {
+ try tty.setColor(w, .yellow);
+ try w.print("{s}", .{@tagName(t)});
+ try tty.setColor(w, .reset);
+ },
+ .type_function => |ty| {
+ try tty.setColor(w, .blue);
+ try w.writeAll("fn(");
+ try tty.setColor(w, .reset);
+ for (ty.param_types, 0..) |param, i| {
+ if (i != 0) {
+ try tty.setColor(w, .blue);
+ try w.writeAll(",");
+ try tty.setColor(w, .reset);
+ }
+ try s.dumpValue(w, Value.fromInterned(param));
+ }
+ try tty.setColor(w, .blue);
+ try w.writeAll(") -> ");
+ try tty.setColor(w, .reset);
+ try s.dumpValue(w, Value.fromInterned(ty.return_type));
+ },
+ inline .float, .int32 => |data, tag| {
+ try tty.setColor(w, .bright_black);
+ try w.writeAll("<");
+ try tty.setColor(w, .blue);
+ try w.writeAll(@tagName(tag));
+ try tty.setColor(w, .reset);
+ try tty.setColor(w, .bright_black);
+ try w.writeAll(", ");
+ try tty.setColor(w, .yellow);
+ try w.print("{d}", .{data});
+ try tty.setColor(w, .bright_black);
+ try w.writeAll(">");
+ try tty.setColor(w, .reset);
+ },
+ .string => |data| {
+ try w.writeAll("<");
+ try tty.setColor(w, .blue);
+ try w.writeAll("string");
+ try tty.setColor(w, .reset);
+ try w.writeAll(", ");
+ try tty.setColor(w, .green);
+ try w.writeAll("\"");
+ try w.writeAll(data);
+ try w.writeAll("\"");
+ try tty.setColor(w, .reset);
+ try w.writeAll(">");
+ },
+ .func => |func| {
+ try tty.setColor(w, .bright_black);
+ try w.writeAll("<");
+ try s.dumpValue(w, Value.fromInterned(func.ty));
+ try tty.setColor(w, .bright_black);
+ try w.writeAll(", ");
+ const decl_name = s.ip.declPtrConst(func.owner_decl).name.toSlice(s.ip);
+ try tty.setColor(w, .bright_green);
+ try w.print("decl '{s}'", .{decl_name});
+ try tty.setColor(w, .bright_black);
+ try w.writeAll(">");
+ try tty.setColor(w, .reset);
+ },
+ }
+ }
+
+ fn dumpInstList(state: *State, writer: anytype, list: []const Inst.Index, l: u8, r: u8) !void {
+ try writer.print(" {c}", .{l});
+ if (list.len > 0) {
+ state.indent += 1;
+
+ for (list) |inst| {
+ try state.newline(writer);
+ try state.dumpInst(writer, inst);
+ }
+
+ state.indent -= 1;
+ try state.newline(writer);
+ }
+ try writer.print("{c}", .{r});
+ }
+
+ fn dumpRefList(state: *State, writer: anytype, list: []const Inst.Ref, l: u8, r: u8) !void {
+ try writer.print(" {c}", .{l});
+ if (list.len > 0) {
+ state.indent += 1;
+
+ for (list) |inst| {
+ try state.newline(writer);
+ try state.dumpRef(writer, inst);
+ }
+
+ state.indent -= 1;
+ try state.newline(writer);
+ }
+ try writer.print("{c}", .{r});
+ }
+};
+
+pub fn format(
+ dump: @This(),
+ comptime unused_fmt: []const u8,
+ _: std.fmt.FormatOptions,
+ writer: anytype,
+) !void {
+ comptime assert(unused_fmt.len == 0);
+
+ var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+ defer arena.deinit();
+
+ const air = dump.air;
+ var state = State{
+ .air = &air,
+ .ip = dump.ip,
+ .tty = detectTTYFromWriter(writer),
+ .arena = arena.allocator(),
+ };
+ try state.dumpInst(writer, .main_block);
+ try writer.writeAll("\n");
+ assert(state.indent == 0);
+}
+
+const std = @import("std");
+const assert = std.debug.assert;
+
+const Air = @import("Air.zig");
+const Inst = Air.Inst;
+
+const Value = @import("Value.zig");
+
+const InternPool = @import("InternPool.zig");
+
+const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter;
diff --git a/src/compiler/Air.zig b/src/compiler/Air.zig
new file mode 100644
index 0000000000000000000000000000000000000000..d27c11cd2f5df0b25f593752b0dba34a6fe0e947
--- /dev/null
+++ b/src/compiler/Air.zig
@@ -0,0 +1,292 @@
+insts: Inst.List.Slice,
+extra: ExtraStore = .{},
+
+pub const ExtraStore = dod.ExtraStore(.air, .@"const");
+pub const EncodedData = dod.EncodedData(Inst.Tag, u64);
+pub const ExtraIndex = ExtraStore.Index;
+
+pub const Inst = struct {
+ tag: Tag,
+ data: EncodedData,
+ flags: Flags,
+
+ pub const List = std.MultiArrayList(Inst);
+
+ pub const Tag = enum {
+ block,
+ js_global_ref,
+ call,
+ @"break",
+ @"if",
+ arg,
+ field,
+
+ // bin
+ add,
+ sub,
+ mul,
+ div,
+
+ eq,
+
+ lt,
+ gt,
+
+ // un
+ negate,
+
+ pub fn Data(comptime tag: Tag) type {
+ return switch (tag) {
+ .block => IndexList,
+ .js_global_ref => InternPool.String.WithLen,
+ .@"break" => Break,
+ .@"if" => If,
+ .call => Call,
+ .arg => Arg,
+ .add,
+ .sub,
+ .mul,
+ .div,
+ .lt,
+ .gt,
+ .eq,
+ => Bin,
+ .negate,
+ => Un,
+ .field,
+ => Field,
+ };
+ }
+ };
+
+ /// reference to either a value or another instruction in this air
+ pub const Ref = packed struct(u32) {
+ kind: Kind,
+ data: u31,
+
+ pub const Kind = enum(u1) { value, inst };
+
+ pub const Decoded = union(enum) {
+ value: Value,
+ inst: Index,
+ };
+
+ pub fn unwrap(ref: Ref) Decoded {
+ return switch (ref.kind) {
+ .value => .{ .value = Value.fromInterned(@enumFromInt(ref.data)) },
+ .inst => .{ .inst = @enumFromInt(ref.data) },
+ };
+ }
+
+ pub fn toOptional(ref: ?Ref) Optional {
+ return if (ref) |r| @enumFromInt(@as(u32, @bitCast(r))) else .none;
+ }
+
+ pub const Optional = enum(u32) {
+ none = std.math.maxInt(u32),
+ _,
+
+ pub fn unwrap(opt: Optional) ?Ref {
+ return if (opt != .none) @bitCast(@intFromEnum(opt)) else null;
+ }
+ };
+ };
+
+ /// reference to another instruction in this air
+ /// note that these cannot point to code in other functions, such makes no sense
+ pub const Index = enum(u32) {
+ main_block,
+ _,
+
+ pub fn toRef(index: Index) Ref {
+ return .{
+ .kind = .inst,
+ .data = @intCast(@intFromEnum(index)),
+ };
+ }
+
+ pub fn toOptional(index: Index) Optional {
+ return @enumFromInt(@intFromEnum(index));
+ }
+
+ pub const Optional = enum(u32) {
+ none = std.math.maxInt(u32),
+ _,
+
+ pub fn unwrap(oi: Optional) ?Index {
+ return if (oi != .none) @enumFromInt(@intFromEnum(oi)) else null;
+ }
+ };
+ };
+};
+
+pub const Flags = packed struct {
+ uses: Uses = .none,
+
+ pub const Uses = enum(u2) {
+ none,
+ single,
+ multi,
+
+ pub fn increment(uses: Uses) Uses {
+ return switch (uses) {
+ .none => .single,
+ .single, .multi => .multi,
+ };
+ }
+ };
+};
+
+pub const IndexList = struct {
+ start: u32,
+ len: u32,
+
+ pub fn slice(self: IndexList, air: *const Air) []const Inst.Index {
+ return @ptrCast(air.extra.list.items[self.start..][0..self.len]);
+ }
+};
+
+pub const RefList = struct {
+ start: u32,
+ len: u32,
+
+ pub fn slice(self: RefList, air: *const Air) []const Inst.Ref {
+ return @ptrCast(air.extra.list.items[self.start..][0..self.len]);
+ }
+};
+
+pub const Break = struct {
+ block_ref: Inst.Index,
+ value: Inst.Ref,
+};
+
+pub const Bin = struct {
+ lhs: Inst.Ref,
+ rhs: Inst.Ref,
+};
+
+pub const Un = struct {
+ operand: Inst.Ref,
+ unused: Unused = .zero,
+};
+
+pub const Field = struct {
+ base: Inst.Ref,
+ field: InternPool.NullTerminatedString,
+};
+
+pub const Call = struct {
+ base: Inst.Ref,
+ call_data: ExtraIndex(Data),
+
+ pub const Data = struct {
+ args: RefList,
+ };
+};
+
+pub const Arg = struct {
+ type: Value,
+ arg_number: u32,
+};
+
+pub const If = struct {
+ cond: Inst.Ref,
+ data: ExtraIndex(Data),
+
+ pub const Data = struct {
+ yes: Inst.Ref,
+ no: Inst.Ref,
+ union_ty: Value,
+ };
+};
+
+pub fn deinit(air: *Air, gpa: std.mem.Allocator) void {
+ defer air.* = undefined;
+ air.insts.deinit(gpa);
+ air.extra.deinit(gpa);
+}
+
+pub fn instTag(air: *const Air, index: Inst.Index) Inst.Tag {
+ return air.insts.items(.tag)[@intFromEnum(index)];
+}
+
+pub fn instFlags(air: *const Air, index: Inst.Index) Flags {
+ return air.insts.items(.flags)[@intFromEnum(index)];
+}
+
+pub fn instEncodedData(air: *const Air, index: Inst.Index) EncodedData {
+ return air.insts.items(.data)[@intFromEnum(index)];
+}
+
+pub fn instData(air: *const Air, comptime tag: Inst.Tag, index: Inst.Index) tag.Data() {
+ return air.instEncodedData(index).decode(tag, air.instTag(index));
+}
+
+pub fn instPayload(air: *const Air, comptime tag: Inst.Tag, index: Inst.Index) tag.Data().T {
+ return air.instData(tag, index).extra_index.get(air);
+}
+
+pub fn typeOf(air: *const Air, ref: Inst.Ref, ip: *InternPool) InternPool.Index {
+ return switch (ref.unwrap()) {
+ .value => |v| v.type(ip).toIntern(),
+ .inst => |i| air.typeOfIndex(i, ip),
+ };
+}
+
+pub fn typeOfIndex(air: *const Air, index: Inst.Index, ip: *InternPool) InternPool.Index {
+ return switch (air.instTag(index)) {
+ .js_global_ref => .opaque_type,
+
+ .block => {
+ const items = air.instData(.block, index).slice(air);
+ return air.typeOfIndex(items[items.len - 1], ip);
+ },
+
+ inline .add, .sub, .mul, .div => |t| air.typeOf(air.instData(t, index).lhs, ip),
+
+ .lt, .gt, .eq => .bool_type,
+
+ .arg => air.instData(.arg, index).type.toIntern(),
+
+ .negate => air.typeOf(air.instData(.negate, index).operand, ip),
+
+ .@"break" => .noreturn_type,
+
+ // todo:
+ .field => .opaque_type,
+
+ .call => {
+ const base_type = air.typeOf(air.instData(.call, index).base, ip);
+ if (base_type == .opaque_type) return .opaque_type;
+ return base_type.toKey(ip).type_function.return_type;
+ },
+
+ .@"if" => air.instData(.@"if", index).data.get(air).union_ty.toIntern(),
+ };
+}
+
+pub fn paramCount(air: Air) u32 {
+ for (air.insts.items(.tag)[1..], 0..) |tag, i| {
+ if (tag != .arg) return @intCast(i);
+ }
+ return @intCast(air.insts.len);
+}
+
+pub fn dumpToWriter(air: Air, ip: *InternPool, writer: anytype) !void {
+ try writer.print("{}", .{Dump{ .air = air, .ip = ip }});
+}
+
+pub const Dump = @import("Air.Dump.zig");
+
+const Air = @This();
+const std = @import("std");
+const mem = std.mem;
+const assert = std.debug.assert;
+
+const Teg = @import("Teg.zig");
+const InternPool = @import("InternPool.zig");
+const dod = @import("dod.zig");
+
+const Unused = enum(u32) { zero = 0 };
+
+const Value = @import("Value.zig");
diff --git a/src/compiler/Ast.Dump.zig b/src/compiler/Ast.Dump.zig
new file mode 100644
index 0000000000000000000000000000000000000000..33186d4e05d9b4ffaa86a9542af1b0698d9ef82c
--- /dev/null
+++ b/src/compiler/Ast.Dump.zig
@@ -0,0 +1,300 @@
+//! A flat array is not very useful for debugging, so this structure implements
+//! a debug formatter for Ast.
+ast: Ast,
+
+const State = struct {
+ ast: *const Ast,
+ indent: usize = 0,
+ node_tags: []const Node.Tag,
+ node_data: []const Node.Data,
+ node_tokens: []const TokenIndex,
+ tty: std.io.tty.Config,
+
+ fn newline(state: State, writer: anytype) !void {
+ try writer.writeByte('\n');
+ if (state.indent > 0)
+ try writer.writeByteNTimes(' ', state.indent * 2);
+ }
+};
+
+pub fn format(
+ dump: Dump,
+ comptime unused_fmt: []const u8,
+ _: std.fmt.FormatOptions,
+ writer: anytype,
+) !void {
+ comptime assert(unused_fmt.len == 0);
+ const ast = dump.ast;
+ var state = State{
+ .ast = &ast,
+ .node_tags = ast.nodes.items(.tag),
+ .node_data = ast.nodes.items(.data),
+ .node_tokens = ast.nodes.items(.main_token),
+ .tty = detectTTYFromWriter(writer),
+ };
+ try dumpNode(writer, &state, 0);
+ try writer.writeAll("\n");
+ assert(state.indent == 0);
+}
+
+fn dumpNode(writer: anytype, state: *State, node: NodeIndex) @TypeOf(writer).Error!void {
+ const tag = state.node_tags[node];
+ const data = state.node_data[node];
+ const data_tag = Node.data_tags[@intFromEnum(tag)];
+ const main_token = state.node_tokens[node];
+ const tty = state.tty;
+
+ try state.tty.setColor(writer, .bright_black);
+ try writer.print("{d}", .{node});
+ try state.tty.setColor(writer, .reset);
+ try writer.writeAll(" ");
+
+ try tty.setColor(writer, .cyan);
+ try writer.print("{s}", .{@tagName(tag)});
+ try tty.setColor(writer, .reset);
+
+ switch (tag) {
+ .builtin_call => {
+ try tty.setColor(writer, .red);
+ try writer.print(" {s}", .{state.ast.tokenSlice(main_token)});
+ try tty.setColor(writer, .reset);
+ },
+ else => {},
+ }
+
+ switch (data_tag) {
+ .none => {
+ switch (tag) {
+ .id => try tty.setColor(writer, .red),
+ .string => try tty.setColor(writer, .green),
+ .number_int, .number_float => try tty.setColor(writer, .yellow),
+ else => {},
+ }
+ switch (tag) {
+ .id,
+ .string,
+ .number_int,
+ .number_float,
+ => try writer.print(" {s}", .{state.ast.tokenSlice(main_token)}),
+
+ else => unreachable,
+ }
+ try tty.setColor(writer, .reset);
+ },
+ .fn_decl => {
+ try writer.writeAll(" ");
+ const fn_decl = state.ast.fullFnDecl(node);
+ if (fn_decl.public) {
+ try writer.writeAll("[pub] ");
+ }
+ try tty.setColor(writer, .magenta);
+ try writer.writeAll(fn_decl.id);
+ try tty.setColor(writer, .reset);
+ try writer.writeAll(" {");
+ {
+ state.indent += 1;
+ defer state.indent -= 1;
+ try state.newline(writer);
+ try writer.writeAll("params:");
+ try dumpNodeList(writer, state, fn_decl.params, '[', ']');
+ if (fn_decl.return_type) |return_type| {
+ try state.newline(writer);
+ try writer.writeAll("return_type: ");
+ try dumpNode(writer, state, return_type);
+ }
+ try state.newline(writer);
+ try writer.writeAll("body: ");
+ try dumpNode(writer, state, fn_decl.body);
+ }
+ try state.newline(writer);
+ try writer.writeAll("}");
+ },
+ .list => {
+ const list = data.list.slice(state.ast);
+ switch (tag) {
+ .root,
+ .block_void,
+ .block_implicit_return,
+ => try dumpNodeList(writer, state, list, '{', '}'),
+ .builtin_call,
+ => try dumpNodeList(writer, state, list, '(', ')'),
+ else => unreachable,
+ }
+ },
+ .var_decl => {
+ try writer.writeAll(" ");
+ const var_decl = state.ast.fullVarDecl(node);
+ if (var_decl.public) {
+ try writer.writeAll("[pub] ");
+ }
+ try writer.writeAll(switch (var_decl.is_const) {
+ true => "constant",
+ false => "mutable",
+ });
+ try tty.setColor(writer, .magenta);
+ try writer.print(" {s}", .{var_decl.id});
+ try tty.setColor(writer, .reset);
+ var indent = false;
+ if (var_decl.type) |type_node| {
+ state.indent += 1;
+ indent = true;
+ try state.newline(writer);
+ try writer.writeAll(": ");
+ try dumpNode(writer, state, type_node);
+ try writer.writeAll(", ");
+ }
+ try writer.writeAll(" = ");
+ try dumpNode(writer, state, var_decl.init);
+ if (indent) {
+ state.indent -= 1;
+ }
+ },
+ .param_decl => {
+ try writer.writeAll(" ");
+ const param_decl = state.ast.fullParamDecl(node);
+ try tty.setColor(writer, .magenta);
+ try writer.writeAll(param_decl.id);
+ try tty.setColor(writer, .reset);
+ try writer.writeAll(": ");
+ try dumpNode(writer, state, param_decl.type);
+ },
+ .bin => {
+ try writer.writeAll(" (");
+ {
+ state.indent += 1;
+ defer state.indent -= 1;
+ try state.newline(writer);
+ try dumpNode(writer, state, data.bin.lhs);
+ try state.newline(writer);
+ try dumpNode(writer, state, data.bin.rhs);
+ }
+ try state.newline(writer);
+ try writer.writeAll(")");
+ },
+ .un => {
+ try writer.writeAll(" (");
+ {
+ state.indent += 1;
+ defer state.indent -= 1;
+ try state.newline(writer);
+ try dumpNode(writer, state, data.un.expr);
+ }
+ try state.newline(writer);
+ try writer.writeAll(")");
+ },
+ .label => {
+ const label_text = state.ast.tokenSlice(main_token);
+ try writer.print(" {s}:", .{label_text});
+
+ try writer.writeAll(" (");
+ {
+ state.indent += 1;
+ defer state.indent -= 1;
+ try state.newline(writer);
+ try dumpNode(writer, state, data.label.expr);
+ }
+ try state.newline(writer);
+ try writer.writeAll(")");
+ },
+ .field => {
+ try writer.writeAll(" (");
+ {
+ state.indent += 1;
+ defer state.indent -= 1;
+ try state.newline(writer);
+ try dumpNode(writer, state, data.field.base);
+ }
+ try state.newline(writer);
+ try writer.writeAll(")");
+ },
+ .@"return" => {
+ if (data.@"return".expr.unwrap()) |expr| {
+ try writer.writeAll(" (");
+ try dumpNode(writer, state, expr);
+ try writer.writeAll(")");
+ }
+ },
+ .@"break" => {
+ const label = data.@"break".label;
+ const value = data.@"break".value;
+ if (label != .none or value != .none) {
+ // try writer.writeAll("(");
+ // try writer.writeAll(")");
+ if (label.unwrap()) |l| {
+ const label_text = state.ast.tokenSlice(l);
+ try writer.print(" :{s}", .{label_text});
+ }
+ if (value.unwrap()) |expr| {
+ try writer.writeAll(" (");
+ try dumpNode(writer, state, expr);
+ try writer.writeAll(")");
+ }
+ }
+ },
+ .call => {
+ const base = data.call.base;
+ const extra = state.ast.extraData(data.call.extra, Call);
+ const arg_list = extra.args.slice(state.ast);
+
+ try writer.writeAll("(");
+ try writer.writeAll("base: ");
+ try dumpNode(writer, state, base);
+ try writer.writeAll(", ");
+ try writer.writeAll("arg:");
+ try dumpNodeList(writer, state, arg_list, '[', ']');
+ try writer.writeAll(")");
+ },
+ .@"if" => {
+ const branch = data.@"if";
+ const if_data = state.ast.extraData(branch.extra, Ast.If);
+
+ try writer.writeAll(" (");
+ try dumpNode(writer, state, branch.cond);
+ try writer.writeAll(") {");
+ {
+ state.indent += 1;
+ defer state.indent -= 1;
+ try state.newline(writer);
+ try writer.writeAll("yes: ");
+ try dumpNode(writer, state, if_data.yes);
+ try state.newline(writer);
+ if (if_data.no.unwrap()) |no| {
+ try writer.writeAll("no: ");
+ try dumpNode(writer, state, no);
+ } else {
+ try writer.writeAll("no: [none]");
+ }
+ }
+ try state.newline(writer);
+ try writer.writeAll("}");
+ },
+ }
+}
+
+fn dumpNodeList(writer: anytype, state: *State, list: []const NodeIndex, l: u8, r: u8) !void {
+ try writer.print(" {c}", .{l});
+ {
+ state.indent += 1;
+ defer state.indent -= 1;
+
+ for (list) |node| {
+ try state.newline(writer);
+ try dumpNode(writer, state, node);
+ }
+ }
+ try state.newline(writer);
+ try writer.print("{c}", .{r});
+}
+
+const Dump = @This();
+
+const std = @import("std");
+const assert = std.debug.assert;
+
+const Ast = @import("Ast.zig");
+const NodeIndex = Ast.NodeIndex;
+const Node = Ast.Node;
+const TokenIndex = Ast.TokenIndex;
+const Call = Ast.Call;
+
+const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter;
diff --git a/src/compiler/Ast.zig b/src/compiler/Ast.zig
new file mode 100644
index 0000000000000000000000000000000000000000..d0f5f446a8bbc5404504c677ee7b59279a5e383e
--- /dev/null
+++ b/src/compiler/Ast.zig
@@ -0,0 +1,701 @@
+//! The Abstract Syntax Tree preserves most details of the source code,
+//! as it is used for formatting. Before semantic analysis, this
+//! structure is normalized into `Teg`, which removes constructs like
+//! grouping, unneded let bindings, and other transforms.
+//!
+//! `ast.nodes` contains nodes stored as three parts:
+//! - tag: the type of the node
+//! - main_token: index to it's primary token, depends on the tag
+//! - data: two u32 values, depends on the tag
+//!
+//! Many nodes encode all their metadata with just `node.data`, using
+//! facts about the language grammar to be able to infer data without
+//! storing it.
+//!
+//! For example, `var_decl` has the following encoding:
+//! - `main_token` is either a `kw_var` or `kw_let`
+//! - the token after `main_token` is always the identifier
+//! - `data.var_decl` contains `NodeIndex`es pointing at the type
+//! and the initializer, which are both expressions.
+//! - If the keyword before `main_token` is `kw_pub` it is a public variable.
+//!
+//! To make decoding these easier, many nodes contain a "full" variant,
+//! such as `full.VarDecl`, which can be aquired with `ast.fullVarDecl(id)`.
+//!
+//! For nodes that must store more than 64 bits of payload data, an "extra"
+//! struct is stored out-of-line in the `extra` field. These can be read
+//! using `ast.extraData(extra_index, T)`. Again, it is recommended to use
+//! the `full` structures with their convenience functions.
+//!
+//! Documentation comments are not stored anywhere, but you can find them
+//! by iterating backwards from a declaration and seeing if a `.doc_comment`
+//! is present. This is provided by `ast.declDocComment(id)`.
+//!
+//! The `Ast` owns token and node data, but not the source code.
+
+/// Externally owned source code.
+utf8_source: []const u8,
+
+tokens: Token.List.Slice,
+nodes: Node.List.Slice,
+extra: []const u32,
+
+pub fn parse(gpa: std.mem.Allocator, source: []const u8) !Ast {
+ const Parse = @import("Parse.zig");
+
+ var todo: ByteOffset = 0;
+ var tokens = try Tokenizer.tokenize(source, gpa, &todo);
+ errdefer tokens.deinit(gpa);
+
+ var parser = Parse.init(gpa, source, tokens);
+ errdefer parser.deinit();
+
+ try parser.nodes.ensureUnusedCapacity(gpa, @divFloor(tokens.len, 4));
+
+ try parser.parseRoot();
+
+ return try parser.finalize(tokens);
+}
+
+pub const Location = struct {
+ line: usize,
+ column: usize,
+ line_start: usize,
+ line_end: usize,
+};
+
+pub const Span = struct {
+ start: u32,
+ end: u32,
+ main: u32,
+};
+
+// This can be removed when the following issue is implemented:
+// See: https://github.com/ziglang/zig/issues/104
+// See: https://github.com/ziglang/zig/issues/3806
+pub const OptionalNodeIndex = enum(u32) {
+ none = 0,
+ _,
+
+ pub fn init(idx: ?NodeIndex) OptionalNodeIndex {
+ assert(idx != 0);
+ return @enumFromInt(idx orelse return .none);
+ }
+
+ pub fn unwrap(idx: OptionalNodeIndex) ?NodeIndex {
+ return switch (idx) {
+ .none => null,
+ _ => @intFromEnum(idx),
+ };
+ }
+};
+
+pub const OptionalTokenIndex = enum(u32) {
+ none = 0,
+ _,
+
+ pub fn init(idx: ?TokenIndex) OptionalTokenIndex {
+ assert(idx != 0);
+ return @enumFromInt(idx orelse return .none);
+ }
+
+ pub fn unwrap(idx: OptionalTokenIndex) ?TokenIndex {
+ return switch (idx) {
+ .none => null,
+ _ => @intFromEnum(idx),
+ };
+ }
+};
+
+pub const OptionalExtraIndex = enum(u32) {
+ none = std.math.maxInt(u32),
+ _,
+
+ pub fn init(idx: ?ExtraIndex) OptionalExtraIndex {
+ assert(idx != std.math.maxInt(u32));
+ return @enumFromInt(idx orelse return .none);
+ }
+
+ pub fn unwrap(idx: OptionalExtraIndex) ?ExtraIndex {
+ return switch (idx) {
+ .none => null,
+ _ => @intFromEnum(idx),
+ };
+ }
+};
+
+pub const Node = struct {
+ tag: Tag,
+ main_token: TokenIndex,
+ data: Data,
+
+ pub const List = std.MultiArrayList(Node);
+
+ pub const Tag = enum {
+ /// Read with `ast.rootNodes()`
+ /// data is .list (containing stmt slice)
+ /// TODO: rename to `module`
+ root,
+ /// main_token is the {
+ /// data is .list (containing stmt slice)
+ ///
+ /// This guarantees that the block either empty, or the last statement
+ /// ends with a semicolon.
+ block_void,
+ /// main_token is the {
+ /// data is .list (containing stmt slice)
+ ///
+ /// This guarantees that the block is not empty and that the last
+ /// statement is an expression statement without a trailing semicolon.
+ block_implicit_return,
+ /// Variable or constant declaration
+ /// `var x: type = init;`
+ /// `let y = x;`
+ /// main_token is the keyword
+ /// data is .var_decl
+ var_decl,
+ /// Function declaration
+ /// `fn foo(x: i32) -> i32 { rhs }`
+ /// Params and return type are stored in extra data
+ fn_decl,
+ /// 'name: type' declaration
+ /// Main token is the identifier
+ param_decl,
+ /// An identifier. main_token is the identifier.
+ id,
+ /// Main token is the data. May not fit in i32 range,
+ /// but will contain no dots or e's.
+ number_int,
+ /// Main token is data. Will contain '.' and/or 'e'.
+ number_float,
+ /// A string literal
+ string,
+ /// A multi-line string literal
+ /// data is .token_list of the tokens, which is guaranteed to be contiguous
+ multiline_string,
+ /// Call to a function
+ /// `foo(x, y);`
+ /// main_token is the `(`
+ /// data is .call
+ call,
+ /// Call to a builtin function
+ /// `@raw("hello");`
+ /// data is .list (containing arg slice)
+ builtin_call,
+ /// `if cond { ... } else ...`
+ /// main_token is the `if`
+ /// data is .@"if"
+ @"if",
+ /// `if let id = ... { ... } else ...`
+ /// main_token is the `if`
+ /// data is .@"if"
+ // if_let_nonvoid,
+ /// `id: ...`
+ /// data is un
+ label,
+ /// `expr.field`
+ /// main_token is the `.`
+ /// data is .field
+ field,
+ /// `lhs[rhs]`
+ /// main_token is the `[`
+ /// data is .bin
+ index,
+ /// `return expr;`
+ /// main_token is the `return`
+ /// data is .@"return"
+ @"return",
+ /// `break :brk expr;`
+ /// main_token is the `break`
+ /// data is .@"break"
+ @"break",
+ /// `(expr)`
+ /// main_token is the `(`
+ /// data is .un
+ group,
+ /// `lhs + rhs`. main_token is the operator.
+ add,
+ /// `lhs - rhs`. main_token is the operator.
+ sub,
+ /// `lhs * rhs`. main_token is the operator.
+ mul,
+ /// `lhs / rhs`. main_token is the operator.
+ div,
+ /// `lhs % rhs`. main_token is the operator.
+ rem,
+ /// `lhs ** rhs`. main_token is the operator.
+ pow,
+ /// `lhs && rhs`. main_token is the operator.
+ @"and",
+ /// `lhs || rhs`. main_token is the operator.
+ @"or",
+ /// `lhs & rhs`. main_token is the operator.
+ bit_and,
+ /// `lhs | rhs`. main_token is the operator.
+ bit_or,
+ /// `lhs ^ rhs`. main_token is the operator.
+ bit_xor,
+ /// `lhs << rhs`. main_token is the operator.
+ shl,
+ /// `lhs >> rhs`. main_token is the operator.
+ shr,
+ /// `lhs == rhs`. main_token is the operator.
+ eq,
+ /// `lhs != rhs`. main_token is the operator.
+ ne,
+ /// `lhs < rhs`. main_token is the operator.
+ lt,
+ /// `lhs <= rhs`. main_token is the operator.
+ le,
+ /// `lhs > rhs`. main_token is the operator.
+ gt,
+ /// `lhs >= rhs`. main_token is the operator.
+ ge,
+ /// `lhs = rhs`. main_token is the operator.
+ assign,
+ /// `lhs += rhs`. main_token is the operator.
+ add_assign,
+ /// `lhs -= rhs`. main_token is the operator.
+ sub_assign,
+ /// `lhs *= rhs`. main_token is the operator.
+ mul_assign,
+ /// `lhs /= rhs`. main_token is the operator.
+ div_assign,
+ /// `lhs %= rhs`. main_token is the operator.
+ rem_assign,
+ /// `lhs **= rhs`. main_token is the operator.
+ pow_assign,
+ /// `lhs &&= rhs`. main_token is the operator.
+ and_assign,
+ /// `lhs ||= rhs`. main_token is the operator.
+ or_assign,
+ /// `lhs &= rhs`. main_token is the operator.
+ bit_and_assign,
+ /// `lhs |= rhs`. main_token is the operator.
+ bit_or_assign,
+ /// `lhs ^= rhs`. main_token is the operator.
+ bit_xor_assign,
+ /// `lhs <<= rhs`. main_token is the operator.
+ shl_assign,
+ /// `lhs >>= rhs`. main_token is the operator.
+ shr_assign,
+ /// `!expr`. main_token is the operator.
+ bool_not,
+ /// `-expr`. main_token is the operator.
+ negate,
+ /// `~expr`. main_token is the operator.
+ bit_not,
+ /// `comptime expr`. main_token is the operator.
+ @"comptime",
+ /// `pure expr`. main_token is the operator.
+ // pure,
+ /// `assert expr`. main_token is the operator.
+ // assert,
+
+ pub fn isAssignBin(op: Tag) bool {
+ return switch (op) {
+ .assign,
+ .add_assign,
+ .sub_assign,
+ .mul_assign,
+ .div_assign,
+ .rem_assign,
+ .pow_assign,
+ .and_assign,
+ .or_assign,
+ .bit_and_assign,
+ .bit_or_assign,
+ .bit_xor_assign,
+ .shl_assign,
+ .shr_assign,
+ => true,
+ .add,
+ .sub,
+ .mul,
+ .div,
+ .rem,
+ .pow,
+ .@"and",
+ .@"or",
+ .bit_and,
+ .bit_or,
+ .bit_xor,
+ .shl,
+ .shr,
+ .eq,
+ .ne,
+ .lt,
+ .le,
+ .gt,
+ .ge,
+ => false,
+ else => unreachable, // not a binary operator
+ };
+ }
+
+ pub fn isBasic(op: Tag) bool {
+ return switch (op) {
+ .id,
+ .string,
+ .number_int,
+ .number_float,
+ .builtin_call,
+ => true,
+ else => false,
+ };
+ }
+ };
+
+ pub const data_tags = std.enums.directEnumArray(Tag, Data.FieldEnum, 0, .{
+ .root = .list,
+ .block_void = .list,
+ .block_implicit_return = .list,
+ .var_decl = .var_decl,
+ .fn_decl = .fn_decl,
+ .param_decl = .param_decl,
+ .id = .none,
+ .number_int = .none,
+ .number_float = .none,
+ .string = .none,
+ .multiline_string = .list,
+ .call = .call,
+ .builtin_call = .list,
+ .@"if" = .@"if",
+ .field = .field,
+ .index = .bin,
+ .@"return" = .@"return",
+ .@"break" = .@"break",
+ .group = .un,
+ .add = .bin,
+ .sub = .bin,
+ .mul = .bin,
+ .div = .bin,
+ .rem = .bin,
+ .pow = .bin,
+ .@"and" = .bin,
+ .@"or" = .bin,
+ .bit_and = .bin,
+ .bit_or = .bin,
+ .bit_xor = .bin,
+ .shl = .bin,
+ .shr = .bin,
+ .eq = .bin,
+ .ne = .bin,
+ .lt = .bin,
+ .le = .bin,
+ .gt = .bin,
+ .ge = .bin,
+ .assign = .bin,
+ .add_assign = .bin,
+ .sub_assign = .bin,
+ .mul_assign = .bin,
+ .div_assign = .bin,
+ .rem_assign = .bin,
+ .pow_assign = .bin,
+ .and_assign = .bin,
+ .or_assign = .bin,
+ .bit_and_assign = .bin,
+ .bit_or_assign = .bin,
+ .bit_xor_assign = .bin,
+ .shl_assign = .bin,
+ .shr_assign = .bin,
+ .bool_not = .un,
+ .negate = .un,
+ .bit_not = .un,
+ .@"comptime" = .un,
+ .label = .label,
+ });
+
+ pub const Data = union {
+ none: struct { Unused = .zero, Unused = .zero },
+ fn_decl: struct {
+ // A `FnDecl`
+ extra: ExtraIndex,
+ body: NodeIndex,
+ },
+ /// either is a node list or a token list, depending on the node's tag
+ list: SubList,
+ var_decl: struct {
+ type: OptionalNodeIndex,
+ init: NodeIndex,
+ },
+ param_decl: struct {
+ type: NodeIndex,
+ unused: Unused = .zero,
+ },
+ bin: struct {
+ lhs: NodeIndex,
+ rhs: NodeIndex,
+ },
+ field: struct {
+ base: NodeIndex,
+ field_token: TokenIndex,
+ },
+ un: struct {
+ expr: NodeIndex,
+ unused: Unused = .zero,
+ },
+ label: struct {
+ expr: NodeIndex,
+ unused: Unused = .zero,
+ },
+ @"return": struct {
+ expr: OptionalNodeIndex,
+ unused: Unused = .zero,
+ },
+ @"break": struct {
+ label: OptionalTokenIndex,
+ value: OptionalNodeIndex,
+ },
+ call: struct {
+ base: NodeIndex,
+ /// A `Call`
+ extra: ExtraIndex,
+ },
+ @"if": struct {
+ cond: NodeIndex,
+ /// A `If`
+ extra: ExtraIndex,
+ },
+
+ const FieldEnum = std.meta.FieldEnum(Data);
+ };
+};
+
+pub const SubList = struct {
+ /// Either NodeIndex or TokenIndex, depending on where this came from.
+ start: u32,
+ len: u32,
+
+ pub fn slice(self: SubList, ast: *const Ast) []const u32 {
+ return ast.extra[self.start..][0..self.len];
+ }
+};
+
+pub const FnDecl = struct {
+ params: SubList,
+ return_type: OptionalNodeIndex,
+};
+
+pub const Call = struct {
+ args: SubList,
+};
+
+pub const If = struct {
+ /// either .block_void or .block_implicit_return
+ yes: NodeIndex,
+ /// if it exists, it is either
+ /// .block_void, .block_implicit_return, or .@"if"
+ no: OptionalNodeIndex,
+};
+
+pub const full = struct {
+ pub const FnDecl = struct {
+ start_token: TokenIndex,
+ public: bool,
+ id: []const u8,
+ params: []const NodeIndex,
+ return_type: ?NodeIndex,
+ body: NodeIndex,
+ };
+
+ pub const VarDecl = struct {
+ start_token: TokenIndex,
+ public: bool,
+ is_const: bool,
+ id: []const u8,
+ type: ?NodeIndex,
+ init: NodeIndex,
+ };
+
+ pub const ParamDecl = struct {
+ start_token: TokenIndex,
+ id: []const u8,
+ type: NodeIndex,
+ };
+};
+
+pub fn deinit(ast: *Ast, gpa: std.mem.Allocator) void {
+ ast.tokens.deinit(gpa);
+ ast.nodes.deinit(gpa);
+ gpa.free(ast.extra);
+ ast.* = undefined;
+}
+
+pub fn nodeTag(ast: Ast, node: NodeIndex) Node.Tag {
+ return ast.nodes.items(.tag)[node];
+}
+
+pub fn nodeData(ast: Ast, node: NodeIndex) Node.Data {
+ return ast.nodes.items(.data)[node];
+}
+
+pub fn nodeToken(ast: Ast, node: NodeIndex) TokenIndex {
+ return ast.nodes.items(.main_token)[node];
+}
+
+pub fn fullFnDecl(ast: Ast, node: NodeIndex) full.FnDecl {
+ const tok_tags = ast.tokens.items(.tag);
+
+ const main_token = ast.nodes.items(.main_token)[node];
+ const public = tok_tags[main_token -| 1] == .kw_pub;
+ assert(tok_tags[main_token] == .kw_fn);
+ assert(tok_tags[main_token + 1] == .id);
+ const id = ast.tokenSlice(main_token + 1);
+
+ const data = ast.nodes.items(.data)[node].fn_decl;
+ const extra = ast.extraData(data.extra, FnDecl);
+
+ return .{
+ .start_token = if (public) main_token - 1 else main_token,
+ .public = public,
+ .id = id,
+ .body = data.body,
+ .params = extra.params.slice(&ast),
+ .return_type = extra.return_type.unwrap(),
+ };
+}
+
+pub fn fullVarDecl(ast: Ast, node: NodeIndex) full.VarDecl {
+ const tok_tags = ast.tokens.items(.tag);
+
+ const main_token = ast.nodes.items(.main_token)[node];
+ const public = tok_tags[main_token -| 1] == .kw_pub;
+ const is_const = switch (tok_tags[main_token]) {
+ .kw_let => true,
+ .kw_var => false,
+ else => |tag| {
+ if (std.debug.runtime_safety) {
+ std.debug.panic("Expected var decl, got {s}", .{@tagName(tag)});
+ }
+ unreachable;
+ },
+ };
+ assert(tok_tags[main_token + 1] == .id);
+ const id = ast.tokenSlice(main_token + 1);
+ const data = ast.nodes.items(.data)[node].var_decl;
+ return .{
+ .start_token = if (public) main_token - 1 else main_token,
+ .public = public,
+ .is_const = is_const,
+ .id = id,
+ .type = data.type.unwrap(),
+ .init = data.init,
+ };
+}
+
+pub fn fullParamDecl(ast: Ast, node: NodeIndex) full.ParamDecl {
+ const tok_tags = ast.tokens.items(.tag);
+
+ const main_token = ast.nodes.items(.main_token)[node];
+ assert(tok_tags[main_token] == .id);
+ const id = ast.tokenSlice(main_token);
+ const data = ast.nodes.items(.data)[node].param_decl;
+ return .{
+ .start_token = main_token,
+ .id = id,
+ .type = data.type,
+ };
+}
+
+pub fn nodeStatements(ast: *const Ast, node: NodeIndex) []const NodeIndex {
+ const data = ast.nodes.items(.data)[node].list;
+ return data.slice(ast);
+}
+
+pub fn tokenLocation(self: Ast, start_offset: ByteOffset, token_index: TokenIndex) Location {
+ var loc = Location{
+ .line = 0,
+ .column = 0,
+ .line_start = start_offset,
+ .line_end = self.source.len,
+ };
+ const token_start = self.tokens.items(.start)[token_index];
+
+ // Scan to by line until we go past the token start
+ while (std.mem.indexOfScalarPos(u8, self.source, loc.line_start, '\n')) |i| {
+ if (i >= token_start) {
+ break; // Went past
+ }
+ loc.line += 1;
+ loc.line_start = i + 1;
+ }
+
+ const offset = loc.line_start;
+ for (self.source[offset..], 0..) |c, i| {
+ if (i + offset == token_start) {
+ loc.line_end = i + offset;
+ while (loc.line_end < self.source.len and self.source[loc.line_end] != '\n') {
+ loc.line_end += 1;
+ }
+ return loc;
+ }
+ if (c == '\n') {
+ loc.line += 1;
+ loc.column = 0;
+ loc.line_start = i + 1;
+ } else {
+ loc.column += 1;
+ }
+ }
+ return loc;
+}
+
+pub fn tokenSlice(ast: Ast, token_index: TokenIndex) []const u8 {
+ return ast.tokens.get(token_index).slice(ast.utf8_source);
+}
+
+pub fn basicNodeSlice(ast: Ast, node: NodeIndex) []const u8 {
+ assert(ast.nodes.items(.tag)[node].isBasic());
+ const token = ast.nodes.items(.main_token)[node];
+ return ast.tokenSlice(token);
+}
+
+pub fn extraData(tree: Ast, index: usize, comptime T: type) T {
+ const fields = std.meta.fields(T);
+ var result: T = undefined;
+ var i: usize = 0;
+ inline for (fields) |field| {
+ @field(result, field.name) = switch (field.type) {
+ NodeIndex => tree.extra[index + i],
+ SubList => SubList{
+ .start = tree.extra[index + i],
+ .len = tree.extra[index + i + 1],
+ },
+ OptionalNodeIndex => @enumFromInt(tree.extra[index + i]),
+ OptionalTokenIndex => @enumFromInt(tree.extra[index + i]),
+ OptionalExtraIndex => @enumFromInt(tree.extra[index + i]),
+ else => @compileError("ExtraData type not supported: " ++ @typeName(field.type)),
+ };
+ i += switch (field.type) {
+ NodeIndex,
+ OptionalNodeIndex,
+ OptionalTokenIndex,
+ OptionalExtraIndex,
+ => 1,
+ SubList => 2,
+ else => comptime unreachable,
+ };
+ }
+ return result;
+}
+
+pub fn dumpToWriter(ast: Ast, writer: anytype) !void {
+ try writer.print("{}", .{Dump{ .ast = ast }});
+}
+
+pub const Dump = @import("Ast.Dump.zig");
+
+pub const Unused = enum(u32) { zero = 0 };
+pub const TokenIndex = u32;
+pub const ExtraIndex = u32;
+pub const NodeIndex = u32;
+
+const Ast = @This();
+const std = @import("std");
+const assert = std.debug.assert;
+
+const Tokenizer = @import("Tokenizer.zig");
+const Token = Tokenizer.Token;
+const TokenList = Tokenizer.Token.List;
+const ByteOffset = Tokenizer.ByteOffset;
diff --git a/src/compiler/Cache.zig b/src/compiler/Cache.zig
new file mode 100644
index 0000000000000000000000000000000000000000..49a4a49d428577d564b2d7495a65c21c68539943
--- /dev/null
+++ b/src/compiler/Cache.zig
@@ -0,0 +1,9 @@
+const std = @import("std");
+
+/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
+pub const bin_digest_len = 16;
+pub const hex_digest_len = bin_digest_len * 2;
+pub const BinDigest = [bin_digest_len]u8;
+pub const HexDigest = [hex_digest_len]u8;
+
+pub const HashHelper = std.Build.Cache.HashHelper;
diff --git a/src/compiler/Comp.zig b/src/compiler/Comp.zig
new file mode 100644
index 0000000000000000000000000000000000000000..7b79942603d4183c3c9f7fe1a463be0ecd89ba98
--- /dev/null
+++ b/src/compiler/Comp.zig
@@ -0,0 +1,395 @@
+//! Compilation
+
+/// General purpose allocator
+gpa: Allocator,
+/// Stores all Type and Value objects.
+intern_pool: InternPool = .{},
+/// All files in the compilation
+files: std.StringArrayHashMapUnmanaged(*File) = .{},
+
+/// # of sema instructions in this compilation cycle
+analyzed_instructions: usize = 0,
+
+// TODO: consider computing this dynamicly per-link
+current_compile_decls: std.ArrayListUnmanaged(Decl.Index) = .{},
+
+pub fn init(comp: *Comp) !void {
+ try comp.intern_pool.init(comp.gpa);
+}
+
+pub fn deinit(comp: *Comp) void {
+ comp.intern_pool.deinit(comp.gpa);
+ for (comp.files.values()) |file| {
+ file.deinit(comp.gpa);
+ }
+ comp.files.deinit(comp.gpa);
+ comp.current_compile_decls.deinit(comp.gpa);
+}
+
+pub fn getFile(comp: *Comp, absolute_file_path: []const u8) !*File {
+ assert(std.fs.path.isAbsolute(absolute_file_path));
+ const gop = try comp.files.getOrPut(comp.gpa, absolute_file_path);
+ if (gop.found_existing) {
+ return gop.value_ptr.*;
+ }
+ const file = try comp.gpa.create(File);
+ file.* = .{
+ .file_path = absolute_file_path,
+ .path_digest = null,
+ .root_module = .none,
+ .status = .never_loader,
+ .source = null,
+ .ast = null,
+ .teg = null,
+ };
+ gop.value_ptr.* = file;
+ return file;
+}
+
+pub const Source = struct {
+ code: [:0]const u8,
+ stat: Stat,
+};
+
+pub const File = struct {
+ /// TODO: change this to a URL
+ file_path: []const u8,
+
+ root_module: Decl.Index.Optional,
+ status: enum {
+ never_loader,
+ /// A failure occurred during loading the file. Retrying might succeed.
+ retryable_failure,
+ /// Errors during parse
+ parse_failure,
+ /// Errors during transform
+ transform_failure,
+ /// Teg is valid
+ success,
+ },
+
+ source: ?Source,
+ ast: ?Ast,
+ teg: ?Teg,
+
+ path_digest: ?Cache.BinDigest,
+
+ pub fn getSource(file: *File, gpa: Allocator) !Source {
+ if (file.source) |source| {
+ return source;
+ }
+
+ // Keep track of inode, file size, mtime, hash so we can detect which files
+ // have been modified when an incremental update is requested.
+ var f = try std.fs.openFileAbsolute(file.file_path, .{});
+ defer f.close();
+
+ const stat = try f.stat();
+
+ if (stat.size > std.math.maxInt(u32))
+ return error.FileTooBig;
+
+ const source = try gpa.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0);
+ errdefer gpa.free(source);
+ const amt = try f.readAll(source);
+ if (amt != stat.size)
+ return error.UnexpectedEndOfFile;
+
+ // Here we do not modify stat fields because this function is the one
+ // used for error reporting. We need to keep the stat fields stale so that
+ // transformFile can know to regenerate Teg.
+
+ const loaded_source = Source{
+ .code = source,
+ .stat = .{
+ .size = stat.size,
+ .inode = stat.inode,
+ .mtime = stat.mtime,
+ },
+ };
+ file.source = loaded_source;
+ return loaded_source;
+ }
+
+ pub fn getAst(file: *File, gpa: Allocator) !Ast {
+ if (file.ast) |ast| {
+ return ast;
+ }
+
+ const ast = try Ast.parse(gpa, (try file.getSource(gpa)).code);
+ file.ast = ast;
+ return ast;
+ }
+
+ pub fn getTeg(file: *File, gpa: Allocator) !Teg {
+ if (file.teg) |teg| {
+ return teg;
+ }
+
+ const bin_digest = hash: {
+ var path_hash: Cache.HashHelper = .{};
+ path_hash.addBytes("0.1.0");
+ path_hash.addBytes(file.file_path);
+ var bin: Cache.BinDigest = undefined;
+ path_hash.hasher.final(&bin);
+ break :hash bin;
+ };
+ file.path_digest = bin_digest;
+
+ const teg = try Transform.transform(try file.getAst(gpa), gpa);
+ file.teg = teg;
+ return teg;
+ }
+
+ pub fn unloadSource(file: *File, gpa: Allocator) void {
+ if (file.source) |source| {
+ gpa.free(source.code);
+ file.source = null;
+ }
+ }
+
+ pub fn unloadAst(file: *File, gpa: Allocator) void {
+ if (file.ast) |*ast| {
+ ast.deinit(gpa);
+ file.ast = null;
+ }
+ }
+
+ pub fn unloadTeg(file: *File, gpa: Allocator) void {
+ if (file.teg) |*teg| {
+ teg.deinit(gpa);
+ file.teg = null;
+ }
+ }
+
+ pub fn deinit(file: *File, gpa: Allocator) void {
+ file.unloadTeg(gpa);
+ file.unloadAst(gpa);
+ file.unloadSource(gpa);
+ gpa.destroy(file);
+ }
+};
+
+pub const Decl = struct {
+ name: InternPool.NullTerminatedString,
+ val: Value,
+ /// The direct parent namespace of the Decl. In the case of the Decl
+ /// corresponding to a file, this is the namespace of the struct, since
+ /// there is no parent.
+ src_namespace: Namespace.Index,
+
+ /// The AST node index of this declaration.
+ /// Must be recomputed when the corresponding source file is modified.
+ // src_node: Ast.NodeIndex,
+ /// Line number corresponding to `src_node`. Stored separately so that source files
+ /// do not need to be loaded into memory in order to compute debug line numbers.
+ /// This value is absolute.
+ // src_line: u32,
+ /// Index of the ZIR `declaration` instruction from which this `Decl` was created.
+ /// For the root `Decl` of a `File` and legacy anonymous decls, this is `.none`.
+ teg_decl_index: InternPool.TrackedInst.Index.Optional,
+
+ /// Represents the "shallow" analysis status. For example, for decls that are functions,
+ /// the function type is analyzed with this set to `in_progress`, however, the semantic
+ /// analysis of the function body is performed with this value set to `success`. Functions
+ /// have their own analysis status field.
+ analysis: enum {
+ /// This Decl corresponds to an AST Node that has not been referenced yet, and therefore
+ /// because of Zig's lazy declaration analysis, it will remain unanalyzed until referenced.
+ unreferenced,
+ /// Semantic analysis for this Decl is running right now.
+ /// This state detects dependency loops.
+ in_progress,
+ /// The file corresponding to this Decl had a parse error or ZIR error.
+ /// There will be a corresponding ErrorMsg in Zcu.failed_files.
+ file_failure,
+ /// This Decl might be OK but it depends on another one which did not
+ /// successfully complete semantic analysis.
+ dependency_failure,
+ /// Semantic analysis failure.
+ /// There will be a corresponding ErrorMsg in Zcu.failed_decls.
+ sema_failure,
+ /// There will be a corresponding ErrorMsg in Zcu.failed_decls.
+ codegen_failure,
+ /// Sematic analysis and constant value codegen of this Decl has
+ /// succeeded. However, the Decl may be outdated due to an in-progress
+ /// update. Note that for a function, this does not mean codegen of the
+ /// function body succeded: that state is indicated by the function's
+ /// `analysis` field.
+ complete,
+ },
+
+ codegen: ?JsTree = null,
+
+ is_pub: bool,
+
+ /// What kind of a declaration is this.
+ kind: Kind,
+
+ pub fn deinit(decl: *Decl, gpa: Allocator) void {
+ defer decl.* = undefined;
+ if (decl.codegen) |*cg| cg.deinit(gpa);
+ }
+
+ pub const Kind = enum {
+ named,
+ };
+
+ pub const Index = InternPool.DeclIndex;
+};
+
+pub const Namespace = struct {
+ parent: Index.Optional,
+ file_scope: *File,
+ /// Direct children of the namespace.
+ /// Declaration order is preserved via entry order.
+ /// These are only declarations named directly by the AST; anonymous
+ /// declarations are not stored here.
+ decls: std.ArrayHashMapUnmanaged(Decl.Index, void, DeclContext, true) = .{},
+
+ pub const DeclContext = struct {
+ comp: *Comp,
+
+ pub fn hash(ctx: @This(), decl_index: Decl.Index) u32 {
+ const decl = ctx.comp.intern_pool.declPtr(decl_index);
+ return std.hash.uint32(@intFromEnum(decl.name));
+ }
+
+ pub fn eql(ctx: @This(), a_decl_index: Decl.Index, b_decl_index: Decl.Index, b_index: usize) bool {
+ _ = b_index;
+ const a_decl = ctx.comp.intern_pool.declPtr(a_decl_index);
+ const b_decl = ctx.comp.intern_pool.declPtr(b_decl_index);
+ return a_decl.name == b_decl.name;
+ }
+ };
+
+ pub const DeclNameAdapter = struct {
+ comp: *Comp,
+
+ pub fn hash(self: @This(), s: InternPool.NullTerminatedString) u32 {
+ _ = self;
+ return std.hash.uint32(@intFromEnum(s));
+ }
+
+ pub fn eql(self: @This(), a: InternPool.NullTerminatedString, b_decl_index: Decl.Index, b_index: usize) bool {
+ _ = b_index;
+ return a == self.comp.intern_pool.declPtr(b_decl_index).name;
+ }
+ };
+
+ pub fn lookupDirect(ns: *Namespace, comp: *Comp, name: InternPool.NullTerminatedString) ?Decl.Index {
+ return ns.decls.getKeyAdapted(name, Namespace.DeclNameAdapter{ .comp = comp });
+ }
+
+ pub const Index = InternPool.NamespaceIndex;
+};
+
+/// Index a module in a Teg.
+/// When a File's Teg is loaded, we call this on the root module.
+/// submodules are analyzed lazily.
+pub fn indexTegModule(comp: *Comp, file: *File, module_index: Teg.Inst.Index) !Namespace.Index {
+ const teg = file.teg.?;
+
+ log.info("Index {} @ {d}", .{ std.fmt.fmtSliceHexLower(file.path_digest.?[0..8]), @intFromEnum(module_index) });
+
+ const ns = try comp.intern_pool.createNamespace(comp.gpa, .{
+ .parent = .none,
+ .file_scope = file,
+ });
+ const ns_ptr = comp.intern_pool.namespacePtr(ns);
+ errdefer comp.intern_pool.destroyNamespace(comp.gpa, ns);
+
+ for (teg.moduleDeclarations(module_index)) |decl_index| {
+ const name = try comp.intern_pool.getOrPutString(
+ comp.gpa,
+ teg.declName(decl_index),
+ .no_embedded_nulls,
+ );
+ const tracked_inst = try comp.intern_pool.trackInst(comp.gpa, file, decl_index);
+ const decl = try comp.intern_pool.createDecl(comp.gpa, .{
+ .name = name,
+ .val = undefined,
+ .src_namespace = ns,
+ .teg_decl_index = tracked_inst.toOptional(),
+ .analysis = .unreferenced,
+ .is_pub = false, // todo:
+ .kind = .named,
+ });
+ try ns_ptr.decls.putNoClobberContext(comp.gpa, decl, void{}, .{ .comp = comp });
+ }
+
+ return ns;
+}
+
+pub fn semaEntryPoint(comp: *Comp, file: *File) !void {
+ const teg = try file.getTeg(comp.gpa);
+
+ const ns = try comp.indexTegModule(file, .main_module_inst);
+ const decl_index = comp.intern_pool.namespacePtr(ns).lookupDirect(comp, .empty) orelse
+ @panic("no top level code?");
+ const decl = comp.intern_pool.declPtr(decl_index);
+
+ assert(comp.current_compile_decls.items.len == 0);
+
+ var arena = std.heap.ArenaAllocator.init(comp.gpa);
+ defer arena.deinit();
+
+ var sema: Sema = .{
+ .comp = comp,
+ .gpa = comp.gpa,
+ .code = teg,
+ .arena = arena.allocator(),
+ .ip = &comp.intern_pool,
+ .decl = decl,
+ .decl_index = decl_index,
+ .namespace_index = ns,
+ };
+ try sema.init();
+ defer sema.deinit();
+ _ = try sema.analyzeDeclaration();
+ try sema.finalize();
+}
+
+pub fn codeGenDecl(c: *Comp, decl: *Decl, air: *const Air) !void {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ std.debug.print("Start Codegen\n", .{});
+ defer std.debug.print("End Codegen\n", .{});
+
+ var arena_state = std.heap.ArenaAllocator.init(c.gpa);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+
+ var cg = try JsNodeGen.init(c.gpa, arena, &c.intern_pool, air);
+ errdefer cg.deinit();
+ try cg.build();
+ decl.codegen = try cg.finalize();
+}
+
+pub fn link(c: *Comp, writer: anytype) !void {
+ try JsLink.linkJsCode(c, c.current_compile_decls.items, writer);
+}
+
+const Comp = @This();
+
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+const assert = std.debug.assert;
+const Stat = std.Build.Cache.File.Stat;
+const log = std.log.scoped(.comp);
+
+const InternPool = @import("InternPool.zig");
+
+const Ast = @import("Ast.zig");
+const Air = @import("Air.zig");
+const Teg = @import("Teg.zig");
+const Cache = @import("Cache.zig");
+const Value = @import("Value.zig");
+const Sema = @import("Sema.zig");
+const Transform = @import("Transform.zig");
+const JsNodeGen = @import("JsNodeGen.zig");
+const JsTree = @import("JsTree.zig");
+const JsLink = @import("JsLink.zig");
+
+const tracer = @import("tracer");
diff --git a/src/compiler/InternPool.zig b/src/compiler/InternPool.zig
new file mode 100644
index 0000000000000000000000000000000000000000..a2a426385d4ca83d4c1e1ebfa446f9327ddee475
--- /dev/null
+++ b/src/compiler/InternPool.zig
@@ -0,0 +1,1018 @@
+//! Large parts of this file are based on the Zig compiler's InternPool.
+//! https://github.com/ziglang/zig/blob/0.12.0/src/InternPool.zig
+
+/// Maps `Key` to `Index`. `Key` objects are not stored anywhere; they are
+/// constructed lazily.
+map: std.AutoArrayHashMapUnmanaged(void, void) = .{},
+items: std.MultiArrayList(Item) = .{},
+extra: ExtraStore = .{},
+
+/// In order to store references to strings in fewer bytes, we copy all
+/// string bytes into here. String bytes can be null. It is up to whomever
+/// is referencing the data here whether they want to store both index and length,
+/// thus allowing null bytes, or store only index, and use null-termination. The
+/// `string_bytes` array is agnostic to either usage.
+string_bytes: std.ArrayListUnmanaged(u8) = .{},
+
+/// Used for finding the index inside `string_bytes`.
+string_table: std.HashMapUnmanaged(
+ u32,
+ void,
+ std.hash_map.StringIndexContext,
+ std.hash_map.default_max_load_percentage,
+) = .{},
+
+/// An index into `tracked_insts` gives a reference to a single ZIR instruction which
+/// persists across incremental updates.
+tracked_insts: std.AutoArrayHashMapUnmanaged(TrackedInst, void) = .{},
+
+/// Rather than allocating Decl objects with an Allocator, we instead allocate
+/// them with this SegmentedList. This provides four advantages:
+/// * Stable memory so that one thread can access a Decl object while another
+/// thread allocates additional Decl objects from this list.
+/// * It allows us to use u32 indexes to reference Decl objects rather than
+/// pointers, saving memory in Type, Value, and dependency sets.
+/// * Using integers to reference Decl objects rather than pointers makes
+/// serialization trivial.
+/// * It provides a unique integer to be used for anonymous symbol names, avoiding
+/// multi-threaded contention on an atomic counter.
+allocated_decls: std.SegmentedList(Comp.Decl, 0) = .{},
+/// When a Decl object is freed from `allocated_decls`, it is pushed into this stack.
+decls_free_list: std.ArrayListUnmanaged(DeclIndex) = .{},
+
+/// Same pattern as with `allocated_decls`.
+allocated_namespaces: std.SegmentedList(Comp.Namespace, 0) = .{},
+/// Same pattern as with `decls_free_list`.
+namespaces_free_list: std.ArrayListUnmanaged(NamespaceIndex) = .{},
+
+/// How many items in the InternPool are statically known.
+/// This is specified with an integer literal and a corresponding comptime
+/// assert below to break an unfortunate and arguably incorrect dependency loop
+/// when compiling.
+pub const static_len = Teg.Inst.Index.static_len;
+comptime {
+ // @compileLog(static_len, static_keys.len);
+ assert(static_len == static_keys.len);
+}
+
+pub const Item = struct {
+ tag: Tag,
+ data: EncodedData,
+
+ pub fn encode(comptime tag: Tag, data: tag.Data()) Item {
+ return .{ .tag = tag, .data = EncodedData.encode(tag, data) };
+ }
+};
+
+pub const ExtraStore = dod.ExtraStore(.intern_pool, .@"var");
+pub const EncodedData = dod.EncodedData(Tag, u32);
+pub const ExtraIndex = ExtraStore.Index;
+
+pub const Tag = enum(u8) {
+ /// This special tag represents a value which was removed from this pool via
+ /// `InternPool.remove`. The item remains allocated to preserve indices, but
+ /// lookups will consider it not equal to any other item, and all queries
+ /// assert not this tag. `data` is unused.
+ removed,
+
+ /// A function type
+ /// data is an extra index into a `FuncType`
+ type_function,
+
+ // --- before this are types ---
+ // --- past this are values ---
+
+ /// A type that can be represented with only an enum tag.
+ /// data is SimpleType enum value.
+ simple_type,
+ /// A value that can be represented with only an enum tag.
+ /// data is SimpleValue enum value.
+ simple_value,
+ /// An int32 value
+ /// data is i32
+ int32,
+ /// A float value
+ /// data is an extra index into an `Float`
+ float,
+ /// A function
+ /// data is an extra index into a `FuncDecl`
+ func,
+ /// A string
+ /// data is an extra index into a `String`
+ string,
+
+ pub fn Data(comptime tag: Tag) type {
+ return switch (tag) {
+ .removed,
+ => unreachable, // does not have a data
+
+ .simple_type => SimpleType,
+ .simple_value => SimpleValue,
+ .int32 => i32,
+
+ else => ExtraIndex(tag.Payload()),
+ };
+ }
+
+ /// When `data` is an extra index, this is the type of the payload.
+ pub fn Payload(comptime tag: Tag) type {
+ return switch (tag) {
+ .removed,
+ .simple_type,
+ .simple_value,
+ .int32,
+ => unreachable, // does not have a payload
+ .float => Float,
+ .func => FuncDecl,
+ .type_function => TypeFunction,
+ .string => String.WithLen,
+ };
+ }
+
+ /// Trailing: N/A
+ pub const FuncDecl = struct {
+ analysis: FuncAnalysis,
+ owner_decl: DeclIndex,
+ ty: Index,
+ teg_body_inst: TrackedInst.Index,
+ };
+
+ /// Trailing:
+ /// 0. param_type: Index for each params_len
+ pub const TypeFunction = struct {
+ params_len: u32,
+ return_type: Index,
+ };
+};
+
+pub const Index = enum(u32) {
+ pub const first_type: Index = .opaque_type;
+ pub const last_type: Index = .noreturn_type;
+ pub const first_value: Index = .void_value;
+ pub const last_value: Index = .empty_string;
+
+ opaque_type,
+ bool_type,
+ int32_type,
+ float_type,
+ string_type,
+ void_type,
+ noreturn_type,
+ type_type,
+
+ void_value,
+ bool_true,
+ bool_false,
+ int32_zero,
+ float_zero,
+ nan,
+ empty_string,
+
+ none = std.math.maxInt(u32),
+ _,
+
+ /// An array of `Index` existing within the `extra` array.
+ /// This type exists to provide a struct with lifetime that is
+ /// not invalidated when items are added to the `InternPool`.
+ pub const Slice = struct {
+ start: u32,
+ len: u32,
+
+ pub fn get(slice: Slice, ip: *const InternPool) []Index {
+ return @ptrCast(ip.extra.list.items[slice.start..][0..slice.len]);
+ }
+ };
+
+ /// Used for a map of `Index` values to the index within a list of `Index` values.
+ const Adapter = struct {
+ indexes: []const Index,
+
+ pub fn eql(ctx: @This(), a: Index, b_void: void, b_map_index: usize) bool {
+ _ = b_void;
+ return a == ctx.indexes[b_map_index];
+ }
+
+ pub fn hash(ctx: @This(), a: Index) u32 {
+ _ = ctx;
+ return std.hash.uint32(@intFromEnum(a));
+ }
+ };
+
+ pub fn toAir(index: Index) Air.Inst.Ref {
+ return .{ .kind = .value, .data = @intCast(@intFromEnum(index)) };
+ }
+
+ pub fn toKey(index: Index, ip: *const InternPool) Key {
+ return ip.indexToKey(index);
+ }
+};
+
+pub const static_keys = [_]Key{
+ SimpleType.@"opaque".key(),
+ SimpleType.bool.key(),
+ SimpleType.int32.key(),
+ SimpleType.float.key(),
+ SimpleType.string.key(),
+ SimpleType.void.key(),
+ SimpleType.noreturn.key(),
+ SimpleType.type.key(),
+
+ SimpleValue.void.key(),
+ SimpleValue.true.key(),
+ SimpleValue.false.key(),
+ .{ .int32 = 0 },
+ .{ .float = 0.0 },
+ .{ .float = Float.pure_nan.decode() },
+ .{ .string = "" },
+};
+
+/// State that is mutable during semantic analysis. This data is not used for
+/// equality or hashing.
+pub const FuncAnalysis = packed struct(u32) {
+ state: State,
+
+ _: u24 = 0,
+
+ pub const State = enum(u8) {
+ /// This function has not yet undergone analysis, because we have not
+ /// seen a potential runtime call. It may be analyzed in future.
+ none,
+ /// Analysis for this function has been queued, but not yet completed.
+ queued,
+ /// This function intentionally only has ZIR generated because it is marked
+ /// inline, which means no runtime version of the function will be generated.
+ inline_only,
+ in_progress,
+ /// There will be a corresponding ErrorMsg in Module.failed_decls
+ sema_failure,
+ /// This function might be OK but it depends on another Decl which did not
+ /// successfully complete semantic analysis.
+ dependency_failure,
+ /// There will be a corresponding ErrorMsg in Module.failed_decls.
+ /// Indicates that semantic analysis succeeded, but code generation for
+ /// this function failed.
+ codegen_failure,
+ /// Semantic analysis and code generation of this function succeeded.
+ success,
+ };
+};
+
+/// An index into `string_bytes`.
+pub const String = enum(u32) {
+ /// An empty string.
+ empty = 0,
+ _,
+
+ pub const WithLen = struct {
+ string: String,
+ len: u32,
+
+ pub fn toSlice(self: WithLen, ip: *const InternPool) []const u8 {
+ return ip.string_bytes.items[@intFromEnum(self.string)..][0..@intCast(self.len)];
+ }
+
+ pub fn tryToNullTerminatedString(string: WithLen, ip: *InternPool, gpa: Allocator) !NullTerminatedString {
+ return string.string.tryToNullTerminatedString(string.len, ip, gpa);
+ }
+
+ pub fn toNullTerminatedString(string: WithLen, ip: *const InternPool) NullTerminatedString {
+ return string.string.tryToNullTerminatedString(string.len, ip);
+ }
+ };
+
+ pub fn toSlice(string: String, len: u64, ip: *const InternPool) []const u8 {
+ return ip.string_bytes.items[@intFromEnum(string)..][0..@intCast(len)];
+ }
+
+ pub fn withLen(string: String, len: u64) WithLen {
+ return .{ .string = string, .len = @intCast(len) };
+ }
+
+ pub fn at(string: String, index: u64, ip: *const InternPool) u8 {
+ return ip.string_bytes.items[@intCast(@intFromEnum(string) + index)];
+ }
+
+ pub fn toNullTerminatedString(string: String, len: u64, ip: *const InternPool) NullTerminatedString {
+ assert(std.mem.indexOfScalar(u8, string.toSlice(len, ip), 0) == null);
+ assert(string.at(len, ip) == 0);
+ return @enumFromInt(@intFromEnum(string));
+ }
+
+ pub fn tryToNullTerminatedString(
+ string: String,
+ len: u64,
+ ip: *InternPool,
+ gpa: Allocator,
+ ) !NullTerminatedString {
+ const slice = string.toSlice(len, ip);
+ if (std.mem.indexOfScalar(u8, slice, 0)) |_| {
+ return error.HasNulls;
+ }
+ if (string.at(len, ip) == 0) {
+ return @enumFromInt(@intFromEnum(string));
+ }
+ return ip.getOrPutString(gpa, slice, .no_embedded_nulls);
+ }
+};
+
+pub const NullTerminatedString = enum(u32) {
+ empty = 0,
+ _,
+
+ pub fn toString(self: NullTerminatedString) String {
+ return @enumFromInt(@intFromEnum(self));
+ }
+
+ // pub fn toOptional(self: NullTerminatedString) Optional{
+ // return @enumFromInt(@intFromEnum(self));
+ // }
+
+ pub fn toSlice(string: NullTerminatedString, ip: *const InternPool) [:0]const u8 {
+ const slice = ip.string_bytes.items[@intFromEnum(string)..];
+ return slice[0..std.mem.indexOfScalar(u8, slice, 0).? :0];
+ }
+
+ pub fn length(string: NullTerminatedString, ip: *const InternPool) u32 {
+ return @intCast(string.toSlice(ip).len);
+ }
+
+ pub fn eqlSlice(string: NullTerminatedString, slice: []const u8, ip: *const InternPool) bool {
+ return std.mem.eql(u8, string.toSlice(ip), slice);
+ }
+
+ const Adapter = struct {
+ strings: []const NullTerminatedString,
+
+ pub fn eql(ctx: @This(), a: NullTerminatedString, b_void: void, b_map_index: usize) bool {
+ _ = b_void;
+ return a == ctx.strings[b_map_index];
+ }
+
+ pub fn hash(ctx: @This(), a: NullTerminatedString) u32 {
+ _ = ctx;
+ return std.hash.uint32(@intFromEnum(a));
+ }
+ };
+
+ const FormatData = struct {
+ string: NullTerminatedString,
+ ip: *const InternPool,
+ };
+ fn format(
+ data: FormatData,
+ comptime specifier: []const u8,
+ _: std.fmt.FormatOptions,
+ writer: anytype,
+ ) @TypeOf(writer).Error!void {
+ const slice = data.string.toSlice(data.ip);
+ if (comptime std.mem.eql(u8, specifier, "")) {
+ try writer.writeAll(slice);
+ } else if (comptime std.mem.eql(u8, specifier, "i")) {
+ try writer.print("{p}", .{std.zig.fmtId(slice)});
+ } else @compileError("invalid format string '" ++ specifier ++ "' for '" ++ @typeName(NullTerminatedString) ++ "'");
+ }
+
+ pub fn fmt(self: NullTerminatedString, ip: *const InternPool) std.fmt.Formatter(format) {
+ return .{ .data = .{ .string = self, .ip = ip } };
+ }
+};
+
+pub const TrackedInst = extern struct {
+ path_digest: Cache.BinDigest,
+ inst: Teg.Inst.Index,
+
+ comptime {
+ // The fields should be tightly packed. See also serialiation logic in `Compilation.saveState`.
+ assert(@sizeOf(@This()) == Cache.bin_digest_len + @sizeOf(Teg.Inst.Index));
+ }
+
+ pub const Index = enum(u32) {
+ _,
+ pub fn resolve(i: TrackedInst.Index, ip: *const InternPool) Teg.Inst.Index {
+ return ip.tracked_insts.keys()[@intFromEnum(i)].inst;
+ }
+ pub fn toOptional(i: TrackedInst.Index) Optional {
+ return @enumFromInt(@intFromEnum(i));
+ }
+ pub const Optional = enum(u32) {
+ none = std.math.maxInt(u32),
+ _,
+ pub fn unwrap(opt: Optional) ?TrackedInst.Index {
+ return switch (opt) {
+ .none => null,
+ _ => @enumFromInt(@intFromEnum(opt)),
+ };
+ }
+ };
+ };
+};
+
+pub const DeclIndex = enum(u32) {
+ _,
+
+ pub fn toOptional(i: DeclIndex) Optional {
+ return @enumFromInt(@intFromEnum(i));
+ }
+
+ pub const Optional = enum(u32) {
+ none = std.math.maxInt(u32),
+ _,
+
+ pub fn init(oi: ?DeclIndex) Optional {
+ return @enumFromInt(@intFromEnum(oi orelse return .none));
+ }
+
+ pub fn unwrap(oi: Optional) ?DeclIndex {
+ if (oi == .none) return null;
+ return @enumFromInt(@intFromEnum(oi));
+ }
+ };
+};
+
+pub const NamespaceIndex = enum(u32) {
+ _,
+
+ pub fn toOptional(i: DeclIndex) Optional {
+ return @enumFromInt(@intFromEnum(i));
+ }
+
+ pub const Optional = enum(u32) {
+ none = std.math.maxInt(u32),
+ _,
+
+ pub fn init(oi: ?NamespaceIndex) Optional {
+ return @enumFromInt(@intFromEnum(oi orelse return .none));
+ }
+
+ pub fn unwrap(oi: Optional) ?NamespaceIndex {
+ if (oi == .none) return null;
+ return @enumFromInt(@intFromEnum(oi));
+ }
+ };
+};
+
+/// A single value captured in the closure of a namespace type. This is not a plain
+/// `Index` because we must differentiate between the following cases:
+/// * runtime-known value (where we store the type)
+/// * comptime-known value (where we store the value)
+/// * decl val (so that we can analyze the value lazily)
+/// * decl ref (so that we can analyze the reference lazily)
+pub const CaptureValue = packed struct(u32) {
+ tag: enum(u2) { @"comptime", runtime, decl_val, decl_ref },
+ idx: u30,
+
+ pub fn wrap(val: Unwrapped) CaptureValue {
+ return switch (val) {
+ .@"comptime" => |i| .{ .tag = .@"comptime", .idx = @intCast(@intFromEnum(i)) },
+ .runtime => |i| .{ .tag = .runtime, .idx = @intCast(@intFromEnum(i)) },
+ .decl_val => |i| .{ .tag = .decl_val, .idx = @intCast(@intFromEnum(i)) },
+ .decl_ref => |i| .{ .tag = .decl_ref, .idx = @intCast(@intFromEnum(i)) },
+ };
+ }
+ pub fn unwrap(val: CaptureValue) Unwrapped {
+ return switch (val.tag) {
+ .@"comptime" => .{ .@"comptime" = @enumFromInt(val.idx) },
+ .runtime => .{ .runtime = @enumFromInt(val.idx) },
+ .decl_val => .{ .decl_val = @enumFromInt(val.idx) },
+ .decl_ref => .{ .decl_ref = @enumFromInt(val.idx) },
+ };
+ }
+
+ pub const Unwrapped = union(enum) {
+ /// Index refers to the value.
+ @"comptime": Index,
+ /// Index refers to the type.
+ runtime: Index,
+ decl_val: DeclIndex,
+ decl_ref: DeclIndex,
+ };
+
+ pub const Slice = struct {
+ start: u32,
+ len: u32,
+ pub fn get(slice: Slice, ip: *const InternPool) []CaptureValue {
+ return @ptrCast(ip.extra.items[slice.start..][0..slice.len]);
+ }
+ };
+};
+
+const SimpleValue = enum {
+ true,
+ false,
+ void,
+
+ pub fn key(self: SimpleValue) Key {
+ return .{ .simple_value = self };
+ }
+};
+
+const SimpleType = enum {
+ @"opaque",
+ int32,
+ float,
+ string,
+ bool,
+ void,
+ noreturn,
+ type,
+
+ pub fn key(self: SimpleType) Key {
+ return .{ .simple_type = self };
+ }
+};
+
+pub const Key = union(enum) {
+ int32: i32,
+ float: f64,
+ string: []const u8,
+ simple_value: SimpleValue,
+ simple_type: SimpleType,
+ type_function: TypeFunction,
+ func: Func,
+
+ const Adapter = struct {
+ intern_pool: *const InternPool,
+
+ pub fn eql(ctx: @This(), a: Key, b_void: void, b_map_index: usize) bool {
+ _ = b_void;
+ if (ctx.intern_pool.items.items(.tag)[b_map_index] == .removed) return false;
+ return ctx.intern_pool.indexToKey(@enumFromInt(b_map_index)).eql(a, ctx.intern_pool);
+ }
+
+ pub fn hash(ctx: @This(), a: Key) u32 {
+ return a.hash32(ctx.intern_pool);
+ }
+ };
+
+ pub const TypeFunction = struct {
+ param_types: []Index,
+ return_type: Index,
+
+ pub fn hash(self: TypeFunction, hasher: *Hash) void {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ for (self.param_types) |param_type| {
+ std.hash.autoHash(hasher, param_type);
+ }
+ std.hash.autoHash(hasher, self.return_type);
+ }
+
+ pub fn eql(a: TypeFunction, b: TypeFunction) bool {
+ return std.mem.eql(Index, a.param_types, b.param_types) and
+ a.return_type == b.return_type;
+ }
+ };
+
+ pub const Func = struct {
+ ty: Index,
+ owner_decl: DeclIndex,
+ teg_body_inst: TrackedInst.Index,
+
+ pub fn hash(self: Func, hasher: *Hash) void {
+ std.hash.autoHash(hasher, self.ty);
+ std.hash.autoHash(hasher, self.owner_decl);
+ std.hash.autoHash(hasher, self.teg_body_inst);
+ }
+ };
+
+ pub fn hash32(key: Key, ip: *const InternPool) u32 {
+ return @truncate(key.hash64(ip));
+ }
+
+ pub fn hash64(key: Key, ip: *const InternPool) u64 {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ _ = ip; // autofix
+ const asBytes = std.mem.asBytes;
+ const KeyTag = @typeInfo(Key).Union.tag_type.?;
+ const seed = @intFromEnum(@as(KeyTag, key));
+
+ return switch (key) {
+ inline .int32,
+ .simple_value,
+ .simple_type,
+ => |x| Hash.hash(seed, asBytes(&x)),
+ .string,
+ => |x| Hash.hash(seed, x),
+ .float => |x| Hash.hash(seed, asBytes(&Float.assertPure(x))),
+ .type_function => |x| {
+ var hasher = Hash.init(seed);
+ x.hash(&hasher);
+ return hasher.final();
+ },
+ .func => |x| {
+ var hasher = Hash.init(seed);
+ x.hash(&hasher);
+ return hasher.final();
+ },
+ };
+ }
+
+ pub fn eql(a: Key, b: Key, ip: *const InternPool) bool {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ _ = ip; // autofix
+ const KeyTag = @typeInfo(Key).Union.tag_type.?;
+ const a_tag: KeyTag = a;
+ const b_tag: KeyTag = b;
+ if (a_tag != b_tag) return false;
+ return switch (a) {
+ inline .int32,
+ .simple_value,
+ .simple_type,
+ => |x, tag| x == @field(b, @tagName(tag)),
+ .string,
+ => |x| std.mem.eql(u8, x, b.string),
+ .float => |x| Float.assertPure(x) == Float.assertPure(b.float),
+ .type_function => |x| x.eql(b.type_function),
+ .func => |a_info| {
+ const b_info = b.func;
+ if (a_info.owner_decl != b_info.owner_decl)
+ return false;
+ if (a_info.ty == b_info.ty)
+ return true;
+
+ // TODO: zig has something extra here for inferred error sets
+ return false;
+ },
+ };
+ }
+};
+
+const EmbeddedNulls = enum {
+ no_embedded_nulls,
+ maybe_embedded_nulls,
+
+ const OptionalNullTerminatedString = @compileError("TODO");
+ const OptionalString = @compileError("TODO");
+
+ fn StringType(comptime embedded_nulls: EmbeddedNulls) type {
+ return switch (embedded_nulls) {
+ .no_embedded_nulls => NullTerminatedString,
+ .maybe_embedded_nulls => String,
+ };
+ }
+
+ fn OptionalStringType(comptime embedded_nulls: EmbeddedNulls) type {
+ return switch (embedded_nulls) {
+ .no_embedded_nulls => OptionalNullTerminatedString,
+ .maybe_embedded_nulls => OptionalString,
+ };
+ }
+};
+
+// implementations:
+
+/// To construct, instantiate `.{ .gpa = gpa }` then call this.
+pub fn init(ip: *InternPool, gpa: Allocator) !void {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ assert(ip.items.len == 0);
+
+ // Reserve string index 0 for an empty string.
+ assert((try ip.getOrPutString(gpa, "", .no_embedded_nulls)) == .empty);
+
+ // So that we can use `catch unreachable` below.
+ try ip.items.ensureUnusedCapacity(gpa, static_keys.len);
+ try ip.map.ensureUnusedCapacity(gpa, static_keys.len);
+ try ip.extra.list.ensureUnusedCapacity(gpa, static_keys.len);
+
+ // This inserts all the statically-known values into the intern pool in the
+ // order expected.
+ for (static_keys) |key| {
+ _ = ip.get(gpa, key) catch unreachable;
+ }
+
+ if (std.debug.runtime_safety) {
+ // Sanity check.
+ assert(ip.indexToKey(.bool_true).simple_value == .true);
+ assert(ip.indexToKey(.bool_false).simple_value == .false);
+ }
+
+ assert(ip.items.len == static_keys.len);
+}
+
+pub fn deinit(ip: *InternPool, gpa: Allocator) void {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ ip.map.deinit(gpa);
+ ip.items.deinit(gpa);
+ ip.extra.deinit(gpa);
+ // ip.limbs.deinit(gpa);
+ ip.string_bytes.deinit(gpa);
+
+ var decls = ip.allocated_decls.iterator(0);
+ while (decls.next()) |decl| {
+ decl.deinit(gpa);
+ }
+
+ ip.allocated_decls.deinit(gpa);
+ ip.decls_free_list.deinit(gpa);
+
+ var namespaces = ip.allocated_namespaces.iterator(0);
+ while (namespaces.next()) |ns| {
+ ns.decls.deinit(gpa);
+ }
+
+ ip.namespaces_free_list.deinit(gpa);
+ ip.allocated_namespaces.deinit(gpa);
+
+ // for (ip.maps.items) |*map| map.deinit(gpa);
+ // ip.maps.deinit(gpa);
+
+ ip.string_table.deinit(gpa);
+
+ ip.tracked_insts.deinit(gpa);
+
+ // ip.src_hash_deps.deinit(gpa);
+ // ip.decl_val_deps.deinit(gpa);
+ // ip.func_ies_deps.deinit(gpa);
+ // ip.namespace_deps.deinit(gpa);
+ // ip.namespace_name_deps.deinit(gpa);
+
+ // ip.first_dependency.deinit(gpa);
+
+ // ip.dep_entries.deinit(gpa);
+ // ip.free_dep_entries.deinit(gpa);
+
+ ip.* = undefined;
+}
+
+pub fn indexToKey(ip: *const InternPool, index: Index) Key {
+ assert(index != .none);
+ const item: Item = ip.items.get(@intFromEnum(index));
+ const data = item.data;
+ return switch (item.tag) {
+ .removed => unreachable,
+
+ .type_function => .{
+ .type_function = ip.extraTypeFunction(data.decode(.type_function, item.tag)),
+ },
+
+ .simple_type => .{ .simple_type = data.decode(.simple_type, item.tag) },
+ .simple_value => .{ .simple_value = data.decode(.simple_value, item.tag) },
+ .int32 => .{ .int32 = data.decode(.int32, item.tag) },
+ .float => .{ .float = data.decode(.float, item.tag).get(ip).decode() },
+ .string => .{ .string = data.decode(.string, item.tag).get(ip).toSlice(ip) },
+ .func => {
+ const func = data.decode(.func, item.tag).get(ip);
+ return .{ .func = .{
+ .ty = func.ty,
+ .owner_decl = func.owner_decl,
+ .teg_body_inst = func.teg_body_inst,
+ } };
+ },
+ };
+}
+
+pub fn get(ip: *InternPool, gpa: Allocator, key: Key) !Index {
+ const t = tracer.traceFmt(@src(), "{s}", .{@tagName(key)});
+ defer t.end();
+
+ const adapter: Key.Adapter = .{ .intern_pool = ip };
+ const gop = try ip.map.getOrPutAdapted(gpa, key, adapter);
+ if (gop.found_existing) return @enumFromInt(gop.index);
+ if (ip.items.len >= std.math.maxInt(u31) - 1) {
+ return error.TooManyValues;
+ }
+
+ const t2 = tracer.traceFmt(@src(), "(insert)", .{});
+ defer t2.end();
+
+ try ip.items.ensureUnusedCapacity(gpa, 1);
+ const item: Item = switch (key) {
+ inline .int32,
+ .simple_type,
+ .simple_value,
+ => |s, tag| Item.encode(@field(Tag, @tagName(tag)), s),
+ .string => |s| Item.encode(
+ .string,
+ try ip.extra.append(
+ gpa,
+ String.WithLen,
+ (try ip.getOrPutString(gpa, s, .maybe_embedded_nulls)).withLen(s.len),
+ ),
+ ),
+ .float => |f| Item.encode(
+ .float,
+ try ip.extra.append(gpa, Float, Float.encode(f)),
+ ),
+ .func => |f| brk: {
+ const extra = try ip.extra.append(gpa, Tag.FuncDecl, .{
+ .analysis = .{
+ .state = .none,
+ },
+ .owner_decl = f.owner_decl,
+ .ty = f.ty,
+ .teg_body_inst = f.teg_body_inst,
+ });
+
+ break :brk Item.encode(.func, extra);
+ },
+ .type_function => |f| brk: {
+ const extra, var trail = try ip.extra.appendTrailing(gpa, Tag.TypeFunction, .{
+ .params_len = @intCast(f.param_types.len),
+ .return_type = f.return_type,
+ }, f.param_types.len);
+ defer trail.end();
+ trail.writeSlice(f.param_types);
+ break :brk Item.encode(.type_function, extra);
+ },
+ };
+ ip.items.appendAssumeCapacity(item);
+ const index: Index = @enumFromInt(ip.items.len - 1);
+ if (std.debug.runtime_safety) {
+ assert(ip.indexToKey(index).eql(key, ip));
+ }
+ return index;
+}
+
+fn extraTypeFunction(
+ ip: *const InternPool,
+ extra_index: ExtraIndex(Tag.TypeFunction),
+) Key.TypeFunction {
+ const type_function, var trail = extra_index.getTrail(ip);
+ const param_types = trail.readInternPoolIndexSlice(type_function.params_len);
+ const result: Key.TypeFunction = .{
+ .return_type = type_function.return_type,
+ .param_types = param_types.get(ip),
+ };
+ return result;
+}
+
+pub fn getOrPutString(
+ ip: *InternPool,
+ gpa: Allocator,
+ slice: []const u8,
+ comptime embedded_nulls: EmbeddedNulls,
+) Allocator.Error!embedded_nulls.StringType() {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ try ip.string_bytes.ensureUnusedCapacity(gpa, slice.len + 1);
+ ip.string_bytes.appendSliceAssumeCapacity(slice);
+ ip.string_bytes.appendAssumeCapacity(0);
+ return ip.getOrPutTrailingString(gpa, slice.len + 1, embedded_nulls);
+}
+
+fn getOrPutTrailingString(
+ ip: *InternPool,
+ gpa: Allocator,
+ len: usize,
+ comptime embedded_nulls: EmbeddedNulls,
+) Allocator.Error!embedded_nulls.StringType() {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ const string_bytes = &ip.string_bytes;
+ const str_index: u32 = @intCast(string_bytes.items.len - len);
+ if (len > 0 and string_bytes.getLast() == 0) {
+ _ = string_bytes.pop();
+ } else {
+ try string_bytes.ensureUnusedCapacity(gpa, 1);
+ }
+ const key: []const u8 = string_bytes.items[str_index..];
+ const has_embedded_null = std.mem.indexOfScalar(u8, key, 0) != null;
+ switch (embedded_nulls) {
+ .no_embedded_nulls => assert(!has_embedded_null),
+ .maybe_embedded_nulls => if (has_embedded_null) {
+ string_bytes.appendAssumeCapacity(0);
+ return @enumFromInt(str_index);
+ },
+ }
+ const gop = try ip.string_table.getOrPutContextAdapted(gpa, key, std.hash_map.StringIndexAdapter{
+ .bytes = string_bytes,
+ }, std.hash_map.StringIndexContext{
+ .bytes = string_bytes,
+ });
+ if (gop.found_existing) {
+ string_bytes.shrinkRetainingCapacity(str_index);
+ return @enumFromInt(gop.key_ptr.*);
+ } else {
+ gop.key_ptr.* = str_index;
+ string_bytes.appendAssumeCapacity(0);
+ return @enumFromInt(str_index);
+ }
+}
+
+pub fn trackInst(ip: *InternPool, gpa: Allocator, file: *Comp.File, inst: Teg.Inst.Index) Allocator.Error!TrackedInst.Index {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ const key: TrackedInst = .{
+ .path_digest = file.path_digest.?,
+ .inst = inst,
+ };
+ const gop = try ip.tracked_insts.getOrPut(gpa, key);
+ return @enumFromInt(gop.index);
+}
+
+pub fn declPtr(ip: *InternPool, index: DeclIndex) *Comp.Decl {
+ return ip.allocated_decls.at(@intFromEnum(index));
+}
+
+pub fn declPtrConst(ip: *const InternPool, index: DeclIndex) *const Comp.Decl {
+ return ip.allocated_decls.at(@intFromEnum(index));
+}
+
+pub fn namespacePtr(ip: *InternPool, index: NamespaceIndex) *Comp.Namespace {
+ return ip.allocated_namespaces.at(@intFromEnum(index));
+}
+
+pub fn createDecl(
+ ip: *InternPool,
+ gpa: Allocator,
+ initialization: Comp.Decl,
+) Allocator.Error!DeclIndex {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ if (ip.decls_free_list.popOrNull()) |index| {
+ ip.allocated_decls.at(@intFromEnum(index)).* = initialization;
+ return index;
+ }
+ const ptr = try ip.allocated_decls.addOne(gpa);
+ ptr.* = initialization;
+ return @enumFromInt(ip.allocated_decls.len - 1);
+}
+
+pub fn destroyDecl(ip: *InternPool, gpa: Allocator, index: DeclIndex) void {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ ip.declPtr(index).* = undefined;
+ ip.decls_free_list.append(gpa, index) catch {
+ // In order to keep `destroyDecl` a non-fallible function, we ignore memory
+ // allocation failures here, instead leaking the Decl until garbage collection.
+ };
+}
+
+pub fn createNamespace(
+ ip: *InternPool,
+ gpa: Allocator,
+ initialization: Comp.Namespace,
+) Allocator.Error!NamespaceIndex {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ if (ip.namespaces_free_list.popOrNull()) |index| {
+ ip.allocated_namespaces.at(@intFromEnum(index)).* = initialization;
+ return index;
+ }
+ const ptr = try ip.allocated_namespaces.addOne(gpa);
+ ptr.* = initialization;
+ return @enumFromInt(ip.allocated_namespaces.len - 1);
+}
+
+pub fn destroyNamespace(ip: *InternPool, gpa: Allocator, index: NamespaceIndex) void {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ ip.namespacePtr(index).* = undefined;
+ ip.namespaces_free_list.append(gpa, index) catch {
+ // In order to keep `destroyNamespace` a non-fallible function, we ignore memory
+ // allocation failures here, instead leaking the Namespace until garbage collection.
+ };
+}
+
+const InternPool = @This();
+
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+const Hash = std.hash.Wyhash;
+const assert = std.debug.assert;
+
+const Comp = @import("Comp.zig");
+const Cache = @import("Cache.zig");
+
+const Teg = @import("Teg.zig");
+const Air = @import("Air.zig");
+
+const dod = @import("dod.zig");
+const Float = dod.Float;
+
+const tracer = @import("tracer");
+
+test InternPool {
+ const testing = std.testing;
+ std.testing.refAllDeclsRecursive(@This());
+
+ var ip_struct: InternPool = .{ .gpa = std.testing.allocator };
+ const ip = &ip_struct;
+ defer ip.deinit();
+
+ const str = try ip.getOrPutString("hello", .no_embedded_nulls);
+ try testing.expectEqualStrings("hello", str.toSlice(ip));
+ const str2 = try ip.getOrPutString("hello", .no_embedded_nulls);
+ try testing.expectEqual(str, str2);
+
+ const int1 = try ip.get(.{ .int32 = 21483 });
+ try testing.expectEqual(@as(i32, 21483), ip.indexToKey(int1).int32);
+ const int2 = try ip.get(.{ .int32 = 21483 });
+ try testing.expectEqual(int1, int2);
+}
diff --git a/src/compiler/JsLink.zig b/src/compiler/JsLink.zig
new file mode 100644
index 0000000000000000000000000000000000000000..2597285951141d69a620e0425bbfdacfa194ebf9
--- /dev/null
+++ b/src/compiler/JsLink.zig
@@ -0,0 +1,457 @@
+/// This is the JavaScript backend's linker.
+/// It takes in a compilation, a list of `JsTree`s and creates a JavaScript
+/// bundle out of them.
+comp: *Comp,
+
+/// Value is an index into `link_strings`
+decls: std.AutoHashMapUnmanaged(Decl.Index, LinkString.Index) = .{},
+link_strings: std.ArrayHashMapUnmanaged(
+ LinkString,
+ GlobalOwner,
+ LinkString.Context,
+ true,
+) = .{},
+string_bytes: std.ArrayListUnmanaged(u8) = .{},
+
+/// The first declaration is the function containing top-level code.
+pub fn linkJsCode(
+ comp: *Comp,
+ decls: []const Decl.Index,
+ writer: anytype,
+) !void {
+ log.info("Linking {d} decls", .{decls.len});
+
+ // Arena allocator is used for the main flow
+ var link: JsLink = .{ .comp = comp };
+ defer link.deinit();
+
+ // Temp allocator is used for extremely temporary string work.
+ // var temp_sfa = std.heap.stackFallback(4096, comp.gpa);
+ // const temp_alloc = temp_sfa.get();
+
+ // 1. Resolve all global identifiers. Function names must not shadow external
+ // names. We also need this list prepared so that local identifier names do not
+ // shadow functions or externals.
+ const root_decl = comp.intern_pool.declPtrConst(decls[0]);
+ assert(root_decl.kind == .named);
+ assert(root_decl.name == .empty);
+
+ for (root_decl.codegen.?.extern_symbols) |extern_symbol| {
+ try link.reserveExternSymbol(extern_symbol);
+ }
+
+ for (decls[1..]) |decl_index| {
+ assert(decl_index != decls[0]);
+
+ const decl = comp.intern_pool.declPtrConst(decl_index);
+
+ // todo: add all externs first
+ for (decl.codegen.?.extern_symbols) |extern_symbol| {
+ try link.reserveExternSymbol(extern_symbol);
+ }
+
+ const name = decl.name.toSlice(&comp.intern_pool);
+ assert(name.len > 0);
+
+ const gop = try link.link_strings.getOrPutContextAdapted(
+ comp.gpa,
+ name,
+ LinkString.SliceContext{ .link = &link },
+ .{ .link = &link },
+ );
+ if (gop.found_existing) {
+ @panic("TODO: handle this shadowing case.");
+ }
+ const name_str = try link.appendLinkString(name);
+ gop.key_ptr.* = name_str;
+ gop.value_ptr.* = .decl;
+ try link.decls.putNoClobber(comp.gpa, decl_index, @enumFromInt(gop.index));
+ }
+
+ try writer.print("// Compiler Demo. Do not use in production\n", .{});
+
+ // 2. Print every function decl
+ for (decls[1..]) |decl_index| {
+ try link.printDecl(decl_index, writer);
+ }
+
+ // 3. Print the top-level code
+ try link.printDeclContents(&root_decl.codegen.?, writer);
+}
+
+fn deinit(link: *JsLink) void {
+ defer link.* = undefined;
+ link.decls.deinit(link.comp.gpa);
+ link.link_strings.deinit(link.comp.gpa);
+ link.string_bytes.deinit(link.comp.gpa);
+}
+
+fn reserveExternSymbol(link: *JsLink, extern_symbol: String.WithLen) !void {
+ const name = extern_symbol.toSlice(&link.comp.intern_pool);
+ const gop = try link.link_strings.getOrPutContextAdapted(
+ link.comp.gpa,
+ name,
+ LinkString.SliceContext{ .link = link },
+ .{ .link = link },
+ );
+
+ if (gop.found_existing) {
+ if (gop.value_ptr.* != .external) {
+ @panic("TODO: Rename Decl");
+ }
+ }
+ gop.key_ptr.* = try link.appendLinkString(name);
+ gop.value_ptr.* = .external;
+}
+
+fn appendLinkString(link: *JsLink, string: []const u8) !LinkString {
+ const start = link.string_bytes.items.len;
+ log.info("Append '{s}' -> .{{ .start = {d}, .len = {d} }}", .{ string, start, string.len });
+ try link.string_bytes.appendSlice(link.comp.gpa, string);
+ return .{ .start = @intCast(start), .len = @intCast(string.len) };
+}
+
+fn declIdentifier(link: *const JsLink, decl: Decl.Index) []const u8 {
+ return (link.decls.get(decl) orelse if (std.debug.runtime_safety) {
+ std.debug.panic("Decl {d} ({}) is not present in link decl map", .{
+ @intFromEnum(decl),
+ link.comp.intern_pool.declPtrConst(decl).name.fmt(&link.comp.intern_pool),
+ });
+ } else unreachable).toString(link).slice(link);
+}
+
+fn printDecl(link: *JsLink, decl_index: Decl.Index, writer: anytype) !void {
+ try writer.writeAll("let ");
+ try writer.writeAll(link.declIdentifier(decl_index));
+ try writer.writeAll(" = ");
+
+ const decl = link.comp.intern_pool.declPtrConst(decl_index);
+ const tree = &decl.codegen.?;
+
+ try writer.writeAll("(");
+ for (0..tree.param_count) |id| {
+ try link.printLocal(@intCast(id), writer);
+ if (id != tree.param_count - 1) try writer.writeAll(", ");
+ }
+ try writer.writeAll(") => {\n");
+ try link.printDeclContents(tree, writer);
+ try writer.writeAll("};\n");
+}
+
+fn printDeclContents(link: *JsLink, tree: *const JsTree, writer: anytype) !void {
+ for (tree.mainStatements()) |stmt| {
+ try link.printExpr(tree, stmt, writer);
+ try writer.writeAll(";\n");
+ }
+}
+
+fn printBlockContents(link: *JsLink, tree: *const JsTree, list: JsTree.IndexList, writer: anytype) !void {
+ assert(list.slice(tree).len > 0);
+ for (list.slice(tree)) |stmt| {
+ try link.printExpr(tree, stmt, writer);
+ switch (tree.nodeTag(stmt)) {
+ else => try writer.writeAll(";"),
+ .stmt_block,
+ .stmt_if,
+ => {},
+ }
+ try writer.writeAll("\n");
+ }
+}
+
+fn printBlock(link: *JsLink, tree: *const JsTree, list: JsTree.IndexList, writer: anytype) !void {
+ if (list.len != 1) {
+ try writer.writeAll("{");
+ try link.printBlockContents(tree, list, writer);
+ try writer.writeAll("}");
+ } else {
+ try link.printBlockContents(tree, list, writer);
+ }
+}
+
+fn printExpr(link: *JsLink, tree: *const JsTree, node: Node.Index, writer: anytype) @TypeOf(writer).Error!void {
+ log.info("print node {d}", .{@intFromEnum(node)});
+ switch (tree.nodeTag(node)) {
+ else => |tag| std.debug.panic("TODO: printExpr .{s}", .{@tagName(tag)}),
+ .root => unreachable,
+
+ .plain_value => {
+ const plain_value: JsTree.PlainValue = tree.nodeData(.plain_value, node);
+ switch (plain_value.value.toKey(&link.comp.intern_pool)) {
+ // TODO: minification
+ .int32 => |val| try writer.print("{d}", .{val}),
+ // TODO: correct printing of many value types
+ .float => |val| try writer.print("{d}", .{val}),
+ .simple_value => |val| try writer.writeAll(switch (val) {
+ .true => "true",
+ .false => "false",
+ // TODO: make codegen decide from null and undefined
+ // instead of emitting void nodes.
+ .void => "undefined",
+ }),
+ // TODO: escapes
+ .string => |val| try writer.print("\"{s}\"", .{val}),
+
+ else => unreachable, // this value is not a plain_value
+ }
+ },
+
+ .local => {
+ const ref: JsTree.LocalRef = tree.nodeData(.local, node);
+ try link.printLocal(ref.id, writer);
+ },
+
+ .global => {
+ const ref: JsTree.GlobalRef = tree.nodeData(.global, node);
+ try writer.writeAll(ref.name.toSlice(&link.comp.intern_pool));
+ },
+
+ .decl => {
+ const ref: JsTree.DeclRef = tree.nodeData(.decl, node);
+ try writer.writeAll(link.declIdentifier(ref.index));
+ },
+
+ .stmt_break => {
+ const stmt: JsTree.StmtBreak = tree.nodeData(.stmt_break, node);
+ try writer.writeAll("break");
+ if (true) {
+ try writer.writeAll(" ");
+ try link.printLabelIdentifier(stmt.label, writer);
+ }
+ },
+
+ .stmt_return => {
+ const stmt: JsTree.StmtReturn = tree.nodeData(.stmt_return, node);
+ try writer.writeAll("return");
+ if (stmt.operand.unwrap()) |operand| {
+ try writer.writeAll(" ");
+ try link.printExpr(tree, operand, writer);
+ }
+ },
+
+ .stmt_declare_local => {
+ const stmt: JsTree.StmtDeclareLocal = tree.nodeData(.stmt_declare_local, node);
+ try writer.writeAll("let ");
+ try link.printLocal(stmt.identifier_index, writer);
+ if (stmt.init.unwrap()) |init| {
+ try writer.writeAll(" = ");
+ try link.printExpr(tree, init, writer);
+ }
+ },
+
+ .stmt_if => {
+ const stmt: JsTree.StmtIf = tree.nodeData(.stmt_if, node);
+ const data = stmt.data.get(tree);
+ try writer.writeAll("if (");
+ try link.printExpr(tree, stmt.cond, writer);
+ try writer.writeAll(") ");
+ if (data.flags.yes_label) {
+ try link.printLabelIdentifier(data.flags.block_label, writer);
+ try writer.writeAll(":");
+ }
+ try link.printBlock(tree, data.yes, writer);
+ if (data.no.len > 0) {
+ try writer.writeAll(" else ");
+ if (data.flags.no_label) {
+ try link.printLabelIdentifier(data.flags.block_label, writer);
+ try writer.writeAll(":");
+ }
+ try link.printBlock(tree, data.no, writer);
+ }
+ },
+
+ .stmt_block => {
+ const stmt: JsTree.StmtBlock = tree.nodeData(.stmt_block, node);
+ try link.printLabelIdentifier(stmt.label, writer);
+ try writer.writeAll(": {\n");
+ try link.printBlockContents(tree, stmt.content.get(tree), writer);
+ try writer.writeAll("}");
+ },
+
+ inline .assign,
+ .add,
+ .sub,
+ .mul,
+ .div,
+ .eq,
+ .lt,
+ .gt,
+ => |tag| {
+ const bin: JsTree.Bin = tree.nodeData(tag, node);
+ const op: Operator = @field(Operator, @tagName(tag));
+ try link.printExpr(tree, bin.lhs, writer);
+ try writer.writeAll(" ");
+ try writer.writeAll(op.symbol());
+ try writer.writeAll(" ");
+ try link.printExpr(tree, bin.rhs, writer);
+ },
+
+ inline .negate => |tag| {
+ const un: JsTree.Un = tree.nodeData(tag, node);
+ const op: Operator = @field(Operator, @tagName(tag));
+ try writer.writeAll(op.symbol());
+ try link.printExpr(tree, un.operand, writer);
+ },
+
+ .ternary => {
+ const ternary: JsTree.Ternary = tree.nodeData(.ternary, node);
+ const data = ternary.data.get(tree);
+ try writer.writeAll("(");
+ try link.printExpr(tree, ternary.cond, writer);
+ try writer.writeAll(" ? ");
+ try link.printExpr(tree, data.yes, writer);
+ try writer.writeAll(" : ");
+ try link.printExpr(tree, data.no, writer);
+ try writer.writeAll(")");
+ },
+
+ .call => {
+ const call: JsTree.Call = tree.nodeData(.call, node);
+ const args = call.args.get(tree);
+
+ try link.printExpr(tree, call.base, writer);
+ try writer.writeAll("(");
+
+ for (args.slice(tree), 0..) |arg, i| {
+ if (i != 0) try writer.writeAll(", ");
+ try link.printExpr(tree, arg, writer);
+ }
+
+ try writer.writeAll(")");
+ },
+
+ .field => {
+ const field: JsTree.Field = tree.nodeData(.field, node);
+
+ try link.printExpr(tree, field.base, writer);
+ try writer.writeAll(".");
+ try writer.writeAll(field.field.toSlice(&link.comp.intern_pool));
+ },
+ }
+}
+
+fn printLocal(link: *JsLink, local: JsTree.LocalIndex, writer: anytype) !void {
+ _ = link; // todo: be smarter about this
+ try writer.print("v{d}", .{local});
+}
+
+fn printLabelIdentifier(link: *JsLink, local: JsTree.LocalIndex, writer: anytype) !void {
+ _ = link; // todo: be smarter about this
+ try writer.print("label{d}", .{local});
+}
+
+const GlobalOwner = enum {
+ external,
+ decl,
+};
+
+// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence
+const Level = enum {
+ comma,
+ assignment,
+ conditional,
+ logical_or,
+ logical_and,
+ bitwise_or,
+ bitwise_xor,
+ bitwise_and,
+ equality,
+ relational,
+ bitshift,
+ additive,
+ multiplicative,
+ exponentiation,
+ prefix,
+ postfix,
+ new,
+ access_and_call,
+ member,
+};
+
+const Operator = enum {
+ assign,
+ add,
+ sub,
+ mul,
+ div,
+ eq,
+ lt,
+ gt,
+ negate,
+
+ pub fn symbol(op: Operator) []const u8 {
+ return switch (op) {
+ .assign => "=",
+ .add => "+",
+ .sub => "-",
+ .mul => "*",
+ .div => "/",
+ .eq => "===",
+ .lt => "<",
+ .gt => ">",
+
+ .negate => "-",
+ };
+ }
+};
+
+const LinkString = struct {
+ start: u32,
+ len: u32,
+
+ fn slice(str: LinkString, link: *const JsLink) []const u8 {
+ return link.string_bytes.items[str.start..][0..str.len];
+ }
+
+ const Index = enum(u32) {
+ _,
+
+ pub fn toString(index: Index, link: *const JsLink) LinkString {
+ return link.link_strings.keys()[@intFromEnum(index)];
+ }
+ };
+
+ const Context = struct {
+ link: *const JsLink,
+
+ pub fn hash(ctx: Context, str: LinkString) u32 {
+ // assert(str.start + str.len < ctx.link.string_bytes.items.len);
+
+ return @truncate(Wyhash.hash(0, str.slice(ctx.link)));
+ }
+
+ pub fn eql(ctx: Context, a: LinkString, b: LinkString, _: usize) bool {
+ assert(a.start + a.len < ctx.link.string_bytes.items.len);
+ assert(b.start + b.len < ctx.link.string_bytes.items.len);
+ return a.start == b.start;
+ }
+ };
+
+ const SliceContext = struct {
+ link: *const JsLink,
+
+ pub fn hash(_: SliceContext, str: []const u8) u32 {
+ return @truncate(Wyhash.hash(0, str));
+ }
+
+ pub fn eql(ctx: SliceContext, a: []const u8, b: LinkString, _: usize) bool {
+ // since the given string is not in string_bytes we do comparisons
+ return std.mem.eql(u8, a, b.slice(ctx.link));
+ }
+ };
+};
+
+const JsLink = @This();
+const std = @import("std");
+const assert = std.debug.assert;
+const Wyhash = std.hash.Wyhash;
+const log = std.log.scoped(.js_link);
+
+const Comp = @import("Comp.zig");
+const Decl = Comp.Decl;
+
+const JsTree = @import("JsTree.zig");
+const Node = JsTree.Node;
+
+const InternPool = @import("InternPool.zig");
+const String = InternPool.String;
diff --git a/src/compiler/JsNodeGen.zig b/src/compiler/JsNodeGen.zig
new file mode 100644
index 0000000000000000000000000000000000000000..c20514022a9fe04e844626d3ecfb3dbdb743bfd0
--- /dev/null
+++ b/src/compiler/JsNodeGen.zig
@@ -0,0 +1,541 @@
+//! Air -> JsTree
+gpa: Allocator,
+arena: Allocator,
+air: *const Air,
+ip: *const InternPool,
+
+nodes: Node.List,
+extra: dod.ExtraStore(.tfr, .@"var"),
+extern_symbols: std.AutoArrayHashMapUnmanaged(InternPool.String.WithLen, void),
+decl_deps: std.AutoArrayHashMapUnmanaged(Decl.Index, void),
+
+param_count: u32,
+air_node_map: []MapEntry,
+scratch: std.ArrayListUnmanaged(u32),
+next_id: u32,
+next_label: u32,
+
+const Error = error{
+ OutOfMemory,
+};
+
+const MapEntry = union {
+ none: void,
+ node: Node.Index,
+ block: packed struct {
+ used: bool = false,
+ index: u31,
+ },
+};
+
+pub fn init(gpa: Allocator, arena: Allocator, ip: *const InternPool, air: *const Air) !JsNodeGen {
+ const air_node_map = try arena.alloc(MapEntry, air.insts.len);
+ if (std.debug.runtime_safety) {
+ @memset(air_node_map, .{ .none = {} });
+ }
+ const param_count = air.paramCount();
+ return .{
+ .gpa = gpa,
+ .arena = arena,
+ .air = air,
+ .ip = ip,
+ .param_count = param_count,
+ .nodes = .{},
+ .extern_symbols = .{},
+ .decl_deps = .{},
+ .air_node_map = air_node_map,
+ .extra = .{},
+ .scratch = .{},
+ .next_id = param_count,
+ .next_label = 0,
+ };
+}
+
+pub fn deinit(c: *JsNodeGen) void {
+ defer c.* = undefined;
+ c.nodes.deinit(c.gpa);
+ c.extra.deinit(c.gpa);
+ c.extern_symbols.deinit(c.gpa);
+ c.decl_deps.deinit(c.gpa);
+
+ // arena can deinit the rest
+}
+
+pub fn finalize(c: *JsNodeGen) !JsTree {
+ // todo: errdefer will probably leave memory in a fully invalid state
+ try c.nodes.setCapacity(c.gpa, c.nodes.len);
+
+ var extra = try c.extra.toOwnedStore(c.gpa);
+ errdefer extra.deinit(c.gpa);
+
+ // todo: i dont like this dupe
+ const extern_symbols = try c.gpa.dupe(InternPool.String.WithLen, c.extern_symbols.keys());
+ errdefer c.gpa.free(extern_symbols);
+
+ const decl_deps = try c.gpa.dupe(Decl.Index, c.decl_deps.keys());
+ errdefer c.gpa.free(decl_deps);
+
+ defer c.deinit();
+
+ return .{
+ .param_count = c.param_count,
+ .nodes = c.nodes.toOwnedSlice(),
+ .extern_symbols = extern_symbols,
+ .decl_deps = decl_deps,
+ .extra = extra,
+ };
+}
+
+pub fn build(c: *JsNodeGen) !void {
+ assert(c.air.instTag(.main_block) == .block);
+ const statements = c.air.instData(.block, .main_block).slice(c.air);
+
+ const scratch_top = c.scratch.items.len;
+ defer c.scratch.shrinkRetainingCapacity(scratch_top);
+
+ _ = try c.add(.root, undefined);
+
+ for (statements[0..c.param_count], 1..) |stmt, expected_id| {
+ assert(@intFromEnum(stmt) == expected_id);
+ assert(c.air.instTag(stmt) == .arg);
+ c.air_node_map[@intFromEnum(stmt)] = .{
+ .node = try c.add(.local, .{ .id = @intFromEnum(stmt) - 1 }),
+ };
+ }
+
+ for (statements[c.param_count..]) |stmt| {
+ try c.codeGenInst(stmt);
+ }
+
+ const items = c.scratch.items[scratch_top..];
+ const top = c.extra.list.items.len;
+ try c.extra.list.appendSlice(c.gpa, items);
+ c.nodes.items(.data)[0] = EncodedData.encode(.root, .{
+ .start = @intCast(top),
+ .len = @intCast(items.len),
+ });
+}
+
+fn codeGenInst(c: *JsNodeGen, inst: Air.Inst.Index) Error!void {
+ std.debug.print("JsNodeGen: #{d}\n", .{@intFromEnum(inst)});
+ switch (c.air.instTag(inst)) {
+ .arg => unreachable, // codegen must skip over these instructions.
+
+ .block => try c.codeGenBlock(inst),
+
+ .js_global_ref => {
+ const data: InternPool.String.WithLen = c.air.instData(.js_global_ref, inst);
+ try c.extern_symbols.put(c.gpa, data, {});
+ c.air_node_map[@intFromEnum(inst)] = .{
+ .node = try c.add(.global, .{ .name = data }),
+ };
+ },
+
+ inline .gt,
+ .lt,
+ .eq,
+ .add,
+ .sub,
+ .mul,
+ .div,
+ => |tag| {
+ const data: Air.Bin = c.air.instData(tag, inst);
+ const node = try c.add(@field(JsTree.Tag, @tagName(tag)), .{
+ .lhs = try c.fetchExpression(data.lhs),
+ .rhs = try c.fetchExpression(data.rhs),
+ });
+ try c.markExpression(inst, node);
+ },
+
+ inline .negate,
+ => |tag| {
+ const data: Air.Un = c.air.instData(tag, inst);
+ const node = try c.add(@field(JsTree.Tag, @tagName(tag)), .{
+ .operand = try c.fetchExpression(data.operand),
+ });
+ try c.markExpression(inst, node);
+ },
+
+ .field => {
+ const data: Air.Field = c.air.instData(.field, inst);
+ const node = try c.add(.field, .{
+ .base = try c.fetchExpression(data.base),
+ .field = data.field,
+ });
+ try c.markExpression(inst, node);
+ },
+
+ .call => {
+ const data: Air.Call = c.air.instData(.call, inst);
+ const call_data = data.call_data.get(c.air);
+
+ try c.extra.list.ensureUnusedCapacity(c.gpa, call_data.args.len);
+ const start = c.extra.list.items.len;
+ c.extra.list.items.len += call_data.args.len;
+
+ const items = c.extra.list.items[start..][0..call_data.args.len];
+
+ for (call_data.args.slice(c.air), items) |arg, *dest| {
+ dest.* = @intFromEnum(try c.fetchExpression(arg));
+ }
+
+ const node = try c.add(.call, .{
+ .base = try c.fetchExpression(data.base),
+ .args = try c.extra.append(c.gpa, JsTree.IndexList, .{
+ .start = @intCast(start),
+ .len = call_data.args.len,
+ }),
+ });
+ try c.markExpression(inst, node);
+ },
+
+ .@"if" => {
+ const data: Air.If = c.air.instData(.@"if", inst);
+ const extra = data.data.get(c.air);
+ var cond = try c.fetchExpression(data.cond);
+
+ if (c.canLowerToExpression(extra.yes) and
+ c.canLowerToExpression(extra.no))
+ {
+ const yes = try c.lowerToExpression(extra.yes);
+ const no = try c.lowerToExpression(extra.yes);
+
+ const ternary = try c.add(.ternary, .{
+ .cond = cond,
+ .data = try c.extra.append(c.gpa, JsTree.Ternary.Data, .{
+ .yes = yes,
+ .no = no,
+ }),
+ });
+
+ try c.markExpression(inst, ternary);
+ } else {
+ const flags = c.air.instFlags(inst);
+ const uses = flags.uses;
+
+ const target = if (uses != .none)
+ try c.addTemporary()
+ else
+ null;
+
+ var yes = try c.codeGenBranch(extra.yes, target);
+ var no = try c.codeGenBranch(extra.no, target);
+ assert(!((yes == null) and (no == null))); // no "void else void" branches
+
+ // when `yes` is void, swap the condition so that the `else` is omitted
+ if (yes == null) {
+ const temp_yes = yes;
+ yes = no;
+ no = temp_yes;
+ cond = try c.add(.bool_not, .{
+ .operand = cond,
+ });
+ }
+
+ const main_expr = try c.add(.stmt_if, .{
+ .cond = cond,
+ .data = try c.extra.append(c.gpa, JsTree.StmtIf.Data, .{
+ .yes = yes.?.list,
+ .no = if (no) |branch|
+ branch.list
+ else
+ .{ .start = 0, .len = 0 },
+ .flags = .{
+ .block_label = @intCast(c.next_label + 1),
+ .yes_label = yes.?.requires_label,
+ .no_label = if (no) |branch| branch.requires_label else false,
+ },
+ }),
+ });
+
+ try c.addStmt(main_expr);
+
+ if (target) |node| {
+ try c.markExpression(inst, node);
+ }
+ }
+ },
+
+ .@"break" => {
+ const brk: Air.Break = c.air.instData(.@"break", inst);
+
+ const operand = try c.fetchExpression(brk.value);
+
+ // break #0 -> return
+ if (brk.block_ref == .main_block) {
+ try c.addStmt(try c.add(.stmt_return, .{
+ .operand = operand.toOptional(),
+ }));
+ } else {
+ const block = &c.air_node_map[@intFromEnum(brk.block_ref)].block;
+
+ const block_insts = c.air.instData(.block, brk.block_ref).slice(c.air);
+ if (block_insts[block_insts.len - 1] != inst) {
+ block.used = true;
+ }
+
+ const assignment: Node.Index.Optional = @enumFromInt(c.scratch.items[block.index]);
+ if (assignment.unwrap()) |target| {
+ try c.addStmt(try c.add(.assign, .{
+ .lhs = target,
+ .rhs = operand,
+ }));
+ }
+
+ try c.addStmt(try c.add(.stmt_break, .{
+ .label = c.scratch.items[block.index + 1],
+ }));
+ }
+ },
+ }
+}
+
+fn markExpression(c: *JsNodeGen, inst: Air.Inst.Index, node: Node.Index) !void {
+ const flags: Air.Flags = c.air.instFlags(inst);
+ const i = @intFromEnum(inst);
+ switch (flags.uses) {
+ .none => {
+ try c.addStmt(node);
+ },
+ .single => {
+ // TODO: this is not aware of side effect fences
+ c.air_node_map[i] = .{ .node = node };
+ },
+ .multi => {
+ const decl = try c.add(.stmt_declare_local, .{
+ .identifier_index = c.next_id,
+ .init = node.toOptional(),
+ });
+ try c.addStmt(decl);
+ const id = try c.add(.local, .{ .id = c.next_id });
+ c.next_id += 1;
+ c.air_node_map[i] = .{ .node = id };
+ },
+ }
+}
+
+fn fetchExpression(c: *JsNodeGen, inst: Air.Inst.Ref) !Node.Index {
+ return switch (inst.unwrap()) {
+ .value => |v| c.valueToExpression(v),
+ .inst => |index| {
+ const data = c.air_node_map[@intFromEnum(index)];
+ if (std.debug.runtime_safety) {
+ const flags: Air.Flags = c.air.instFlags(index);
+ switch (flags.uses) {
+ .none => unreachable, // incorrectly marked instruction as unused,
+ .single => {},
+ .multi => assert(c.nodes.items(.tag)[@intFromEnum(data.node)] == .local),
+ }
+ }
+ return data.node;
+ },
+ };
+}
+
+fn valueToExpression(c: *JsNodeGen, v: Value) !Node.Index {
+ const intern_tag = c.ip.items.items(.tag)[@intFromEnum(v.toIntern())];
+ switch (intern_tag) {
+ .removed => unreachable, // UAF
+
+ .type_function, .simple_type => unreachable, // Type values are not allowed here
+
+ .int32, .float, .string, .simple_value => {
+ return try c.add(.plain_value, .{ .value = v });
+ },
+
+ // funcs always point to decls
+ .func => {
+ const func = v.toKey(c.ip).func;
+ try c.decl_deps.put(c.gpa, func.owner_decl, {});
+ return try c.add(.decl, .{ .index = func.owner_decl });
+ },
+ }
+}
+
+fn addTemporary(c: *JsNodeGen) !Node.Index {
+ try c.addStmt(try c.add(.stmt_declare_local, .{
+ .identifier_index = c.next_id,
+ .init = .none,
+ }));
+ const id = try c.add(.local, .{ .id = c.next_id });
+ c.next_id += 1;
+ return id;
+}
+
+const Branch = struct {
+ requires_label: bool,
+ list: JsTree.IndexList,
+};
+
+fn codeGenBranch(c: *JsNodeGen, inst: Air.Inst.Ref, temporary: ?Node.Index) !?Branch {
+ const scratch_top = c.scratch.items.len;
+ defer c.scratch.shrinkRetainingCapacity(scratch_top);
+
+ c.next_label += 1;
+ defer c.next_label -= 1;
+ const this_label = c.next_label;
+
+ var requires_label = false;
+ var items = switch (inst.unwrap()) {
+ .value => |v| items: {
+ if (v.ip_index == .void_value)
+ return null;
+ if (temporary) |temp| {
+ const node = try c.valueToExpression(v);
+ try c.addStmt(try c.add(.assign, .{
+ .lhs = temp,
+ .rhs = node,
+ }));
+ }
+ break :items c.scratch.items[scratch_top..];
+ },
+ .inst => |i| items: {
+ assert(c.air.instTag(i) == .block);
+ c.air_node_map[@intFromEnum(i)] = .{
+ .block = .{ .index = @intCast(scratch_top) },
+ };
+
+ try c.scratch.appendSlice(c.arena, &.{
+ @intFromEnum(if (temporary) |temp| temp.toOptional() else .none),
+ this_label,
+ });
+
+ std.debug.print("requires_label {}\n", .{requires_label});
+ const block: Air.IndexList = c.air.instData(.block, i);
+ for (block.slice(c.air)) |item| {
+ try c.codeGenInst(item);
+ }
+
+ requires_label = c.air_node_map[@intFromEnum(i)].block.used;
+
+ break :items c.scratch.items[scratch_top + 2 ..];
+ },
+ };
+
+ const last_node = c.scratch.items[c.scratch.items.len - 1];
+ if (c.nodes.items(.tag)[last_node] == .stmt_break and
+ c.nodes.items(.data)[last_node].decode(.stmt_break, .stmt_break).label == this_label)
+ {
+ items.len -= 1;
+ c.nodes.len -= 1;
+ }
+
+ const top = c.extra.list.items.len;
+ try c.extra.list.appendSlice(c.gpa, items);
+ return .{
+ .requires_label = requires_label,
+ .list = .{
+ .start = @intCast(top),
+ .len = @intCast(items.len),
+ },
+ };
+}
+
+fn codeGenBlock(c: *JsNodeGen, inst: Air.Inst.Index) !void {
+ const scratch_top = c.scratch.items.len;
+ errdefer c.scratch.shrinkRetainingCapacity(scratch_top);
+
+ c.next_label += 1;
+ defer c.next_label -= 1;
+
+ const next_label = c.next_label;
+
+ try c.scratch.appendSlice(c.arena, &.{
+ // @intFromEnum(if (temporary) |temp| temp.toOptional() else .none),
+ @intFromEnum(Node.Index.Optional.none),
+ next_label,
+ });
+
+ c.air_node_map[@intFromEnum(inst)] = .{
+ .block = .{ .index = @intCast(scratch_top) },
+ };
+ const block: Air.IndexList = c.air.instData(.block, inst);
+
+ for (block.slice(c.air)) |item| {
+ try c.codeGenInst(item);
+ }
+
+ const items = c.scratch.items[scratch_top + 2 ..];
+ const top = c.extra.list.items.len;
+ try c.extra.list.appendSlice(c.gpa, items);
+ c.scratch.shrinkRetainingCapacity(scratch_top);
+ try c.addStmt(try c.add(.stmt_block, .{
+ .label = c.next_label,
+ .content = try c.extra.append(c.gpa, JsTree.IndexList, .{
+ .start = @intCast(top),
+ .len = @intCast(items.len),
+ }),
+ }));
+}
+
+fn canLowerToExpression(c: *const JsNodeGen, inst: Air.Inst.Ref) bool {
+ return switch (inst.unwrap()) {
+ .value => true,
+ .inst => |index| switch (c.air.instTag(index)) {
+ .js_global_ref,
+ .call,
+ .arg,
+ .field,
+ .add,
+ .sub,
+ .mul,
+ .div,
+ .eq,
+ .lt,
+ .gt,
+ .negate,
+ => true,
+ .@"break",
+ => false,
+ // TODO: these depend
+ .block => false,
+ .@"if" => false,
+ },
+ };
+}
+
+fn lowerToExpression(c: *JsNodeGen, ref: Air.Inst.Ref) !Node.Index {
+ assert(c.canLowerToExpression(ref));
+ return switch (ref.unwrap()) {
+ .value => |v| try c.valueToExpression(v),
+ .inst => |index| {
+ try c.codeGenInst(index);
+ return c.fetchExpression(ref);
+ },
+ };
+}
+
+fn add(c: *JsNodeGen, comptime tag: JsTree.Tag, data: tag.Data()) !Node.Index {
+ try c.nodes.append(c.gpa, .{
+ .tag = tag,
+ .data = EncodedData.encode(tag, data),
+ });
+ return @enumFromInt(c.nodes.len - 1);
+}
+
+fn addStmt(c: *JsNodeGen, stmt: Node.Index) !void {
+ try c.scratch.append(c.arena, @intFromEnum(stmt));
+}
+
+const JsNodeGen = @This();
+
+const std = @import("std");
+const assert = std.debug.assert;
+const Allocator = std.mem.Allocator;
+
+const InternPool = @import("InternPool.zig");
+const NullTerminatedString = InternPool.NullTerminatedString;
+
+const Comp = @import("Comp.zig");
+const Decl = Comp.Decl;
+
+const JsTree = @import("JsTree.zig");
+const EncodedData = JsTree.EncodedData;
+const Node = JsTree.Node;
+const LocalIndex = JsTree.LocalIndex;
+
+const Air = @import("Air.zig");
+
+const Value = @import("Value.zig");
+
+const dod = @import("dod.zig");
diff --git a/src/compiler/JsTree.zig b/src/compiler/JsTree.zig
new file mode 100644
index 0000000000000000000000000000000000000000..dbf57341c79c0bac4a1fa32fdbb55734b27b83ec
--- /dev/null
+++ b/src/compiler/JsTree.zig
@@ -0,0 +1,245 @@
+//! JavaScript Function Representation is the last representation before javascript
+//! source bytes are created. One notable thing here is that all local identifiers
+//! are numbered, instead of being named. The first `param_count` local identifiers
+//! refer to parameters, and the rest refer to the respective block that defined them.
+
+param_count: u32,
+nodes: Node.List.Slice,
+extern_symbols: []String.WithLen,
+decl_deps: []Decl.Index,
+extra: ExtraStore,
+
+pub const LocalIndex = u32;
+pub const ExtraStore = dod.ExtraStore(.tfr, .@"const");
+pub const EncodedData = dod.EncodedData(Tag, u64);
+pub const ExtraIndex = ExtraStore.Index;
+
+pub const Tag = enum {
+ root,
+ /// Only allow simple value types: int, float, string, bool
+ plain_value,
+ /// Reference to a local identifier (give an VarId)
+ local,
+ /// Reference to a global identifier (give a NullTerminatedString)
+ global,
+ /// Reference to a declaration (contains a Decl.Index)
+ decl,
+ // bin ops
+ assign,
+ add,
+ sub,
+ mul,
+ div,
+ eq,
+ lt,
+ gt,
+ // exprs
+ call,
+ field,
+ ternary,
+ // un ops
+ negate,
+ bool_not,
+ // stmts
+ /// `let ... = ...`
+ stmt_declare_local,
+ /// `if (...) ... else ...`
+ stmt_if,
+ /// `break ... ...`
+ stmt_break,
+ /// `return ...` (break from block #0)
+ stmt_return,
+ /// `...: { ... }`
+ stmt_block,
+
+ pub fn Data(comptime tag: Tag) type {
+ return switch (tag) {
+ .root => IndexList,
+ .plain_value => PlainValue,
+ .local => LocalRef,
+ .global => GlobalRef,
+ .decl => DeclRef,
+ .assign,
+ .add,
+ .sub,
+ .mul,
+ .div,
+ .eq,
+ .lt,
+ .gt,
+ => Bin,
+ .ternary => Ternary,
+ .negate,
+ .bool_not,
+ => Un,
+ .call => Call,
+ .field => Field,
+ .stmt_if => StmtIf,
+ .stmt_break => StmtBreak,
+ .stmt_return => StmtReturn,
+ .stmt_declare_local => StmtDeclareLocal,
+ .stmt_block => StmtBlock,
+ };
+ }
+};
+
+pub const Node = struct {
+ tag: Tag,
+ data: EncodedData,
+
+ pub const List = std.MultiArrayList(Node);
+
+ pub const Index = enum(u32) {
+ root = 0,
+ _,
+
+ pub fn toOptional(i: Index) Optional {
+ return @enumFromInt(@intFromEnum(i));
+ }
+
+ pub const Optional = enum(u32) {
+ none = std.math.maxInt(u32),
+ _,
+
+ pub fn unwrap(oi: Optional) ?Index {
+ return if (oi == .none)
+ null
+ else
+ @enumFromInt(@intFromEnum(oi));
+ }
+ };
+ };
+};
+
+pub const PlainValue = struct {
+ value: Value,
+ unused: Unused = .zero,
+};
+
+pub const LocalRef = struct {
+ id: LocalIndex,
+ unused: Unused = .zero,
+};
+
+pub const GlobalRef = struct {
+ name: InternPool.String.WithLen,
+};
+
+pub const DeclRef = struct {
+ index: Decl.Index,
+ unused: Unused = .zero,
+};
+
+pub const StmtDeclareLocal = struct {
+ identifier_index: u32,
+ init: Node.Index.Optional,
+};
+
+pub const Bin = struct {
+ lhs: Node.Index,
+ rhs: Node.Index,
+};
+
+pub const Un = struct {
+ operand: Node.Index,
+ unused: Unused = .zero,
+};
+
+pub const Call = struct {
+ base: Node.Index,
+ args: ExtraIndex(IndexList),
+};
+
+pub const Field = struct {
+ base: Node.Index,
+ field: NullTerminatedString,
+};
+
+pub const IndexList = struct {
+ start: u32,
+ len: u32,
+
+ pub fn slice(list: IndexList, tree: *const JsTree) []const Node.Index {
+ return @ptrCast(tree.extra.list.items[list.start..][0..list.len]);
+ }
+};
+
+pub const StmtIf = struct {
+ cond: Node.Index,
+ data: ExtraIndex(Data),
+
+ pub const Data = struct {
+ yes: IndexList,
+ no: IndexList,
+ flags: Flags,
+ };
+
+ pub const Flags = packed struct(u32) {
+ block_label: u16,
+ yes_label: bool,
+ no_label: bool,
+ _: u14 = 0,
+ };
+};
+
+pub const Ternary = struct {
+ cond: Node.Index,
+ data: ExtraIndex(Data),
+
+ pub const Data = struct {
+ yes: Node.Index,
+ no: Node.Index,
+ };
+};
+
+pub const StmtBreak = struct {
+ label: u32,
+ unused: Unused = .zero,
+};
+
+pub const StmtReturn = struct {
+ operand: Node.Index.Optional,
+ unused: Unused = .zero,
+};
+
+pub const StmtBlock = struct {
+ label: u32,
+ content: ExtraIndex(IndexList),
+};
+
+pub fn deinit(tree: *JsTree, gpa: std.mem.Allocator) void {
+ tree.nodes.deinit(gpa);
+ gpa.free(tree.decl_deps);
+ gpa.free(tree.extern_symbols);
+ tree.extra.deinit(gpa);
+}
+
+pub fn nodeTag(tree: JsTree, index: Node.Index) Tag {
+ return tree.nodes.items(.tag)[@intFromEnum(index)];
+}
+
+pub fn nodeData(tree: JsTree, comptime tag: Tag, index: Node.Index) tag.Data() {
+ return tree.nodes.items(.data)[@intFromEnum(index)].decode(tag, tree.nodeTag(index));
+}
+
+pub fn mainStatements(tree: *const JsTree) []const Node.Index {
+ return tree.nodeData(.root, .root).slice(tree);
+}
+
+const JsTree = @This();
+
+const std = @import("std");
+const assert = std.debug.assert;
+
+const InternPool = @import("InternPool.zig");
+const NullTerminatedString = InternPool.NullTerminatedString;
+const String = InternPool.String;
+
+const Comp = @import("Comp.zig");
+const Decl = Comp.Decl;
+
+const dod = @import("dod.zig");
+
+const Value = @import("Value.zig");
+
+const Unused = @import("Ast.zig").Unused;
diff --git a/src/compiler/Parse.zig b/src/compiler/Parse.zig
new file mode 100644
index 0000000000000000000000000000000000000000..6dc6147ebdfb1c0fc4218b36d46beef745d789c9
--- /dev/null
+++ b/src/compiler/Parse.zig
@@ -0,0 +1,961 @@
+//! Parser takes a Token list and creates a Node list. It does not own the source code or
+//! token data, however transferring into an `Ast` does move token list ownership.
+gpa: Allocator,
+source: []const u8,
+token_tags: []const Token.Tag,
+token_starts: []const ByteOffset,
+
+pos: TokenIndex = 0,
+nodes: Node.List = .{},
+extra: ArrayListUnmanaged(u32) = .{},
+scratch: ArrayListUnmanaged(NodeIndex) = .{},
+
+ast_error: ?AstError = null,
+
+pub fn init(gpa: Allocator, source: []const u8, tokens: Token.List.Slice) Parse {
+ return Parse{
+ .gpa = gpa,
+ .source = source,
+ .token_tags = tokens.items(.tag),
+ .token_starts = tokens.items(.offset),
+ };
+}
+
+/// This transfers the parse result into an Ast
+pub fn finalize(p: *Parse, token_slice: Token.List.Slice) !Ast {
+ const extra = try p.extra.toOwnedSlice(p.gpa);
+ defer p.* = undefined;
+ assert(token_slice.items(.tag).ptr == p.token_tags.ptr);
+ assert(token_slice.items(.offset).ptr == p.token_starts.ptr);
+ p.scratch.deinit(p.gpa);
+
+ const ast: Ast = .{
+ .utf8_source = p.source,
+ .tokens = token_slice,
+ .nodes = p.nodes.toOwnedSlice(),
+ .extra = extra,
+ };
+
+ ast.dumpToWriter(std.io.getStdErr().writer()) catch {};
+
+ return ast;
+}
+
+/// This discards all parser work
+pub fn deinit(p: *Parse) void {
+ defer p.* = undefined;
+ p.nodes.deinit(p.gpa);
+ p.extra.deinit(p.gpa);
+ p.scratch.deinit(p.gpa);
+}
+
+pub const Error = error{
+ SyntaxError,
+};
+
+/// When error.SyntaxError is returned, `p.err` is set to this as well as `p.pos`
+pub const AstError = union(enum) {
+ expected_token: Token.Tag,
+ expected_statement,
+ expected_expr,
+ expected_else_value,
+};
+
+pub fn parseRoot(p: *Parse) !void {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ assert(p.nodes.len == 0);
+ assert(p.extra.items.len == 0);
+ assert(p.pos == 0);
+ assert(p.nodes.capacity > 0);
+
+ p.nodes.appendAssumeCapacity(.{
+ .tag = .root,
+ .main_token = 0,
+ .data = undefined,
+ });
+
+ const list = try p.parseModuleMembers();
+ p.nodes.items(.data)[0] = .{
+ .list = list,
+ };
+}
+
+fn parseModuleMembers(p: *Parse) !SubList {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ const scratch_top = p.scratch.items.len;
+ defer p.scratch.shrinkRetainingCapacity(scratch_top);
+
+ // skip module doc comment
+ while (p.eatToken(.module_doc_comment)) |_| {}
+
+ while (true) switch (p.currentTag()) {
+ .kw_let,
+ .kw_var,
+ .kw_fn,
+ .kw_pub,
+ => try p.scratch.append(p.gpa, try p.parseModuleDecl()),
+
+ .kw_struct,
+ .kw_enum,
+ .kw_trait,
+ .kw_impl,
+ => return error.Todo,
+
+ .eof => break,
+
+ else => {
+ if (!p.isExpressionStart()) {
+ return p.fail(.expected_statement);
+ }
+
+ try p.scratch.append(p.gpa, try p.parseExpression());
+ _ = try p.expect(.semi);
+ },
+ };
+
+ return p.scratchToSubList(scratch_top);
+}
+
+fn parseModuleDecl(p: *Parse) !NodeIndex {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ _ = p.eatToken(.kw_pub);
+
+ return switch (p.currentTag()) {
+ .kw_var, .kw_let => try p.parseVarDecl(),
+
+ .kw_fn => try p.parseFnDecl(),
+
+ else => unreachable,
+ };
+}
+
+fn parseFnDecl(p: *Parse) !NodeIndex {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ const main_token = p.assertToken(.kw_fn);
+
+ _ = try p.expect(.id);
+
+ _ = try p.expect(.start_paren);
+
+ const params = try p.parseParamList();
+
+ const return_type = if (p.eatToken(.minus_lt)) |_|
+ try p.parseExpression()
+ else
+ null;
+
+ try p.expectWithoutAdvance(.start_brace);
+ const body = try p.parseBlock();
+
+ const extra = try p.addExtra(Ast.FnDecl, .{
+ .params = params,
+ .return_type = optionalNodeIndex(return_type),
+ });
+
+ return try p.addNode(.{
+ .tag = .fn_decl,
+ .main_token = main_token,
+ .data = .{ .fn_decl = .{
+ .body = body,
+ .extra = extra,
+ } },
+ });
+}
+
+fn parseParamList(p: *Parse) !SubList {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ const scratch_top = p.scratch.items.len;
+ defer p.scratch.shrinkRetainingCapacity(scratch_top);
+
+ while (true) {
+ const trace_param = t.traceFmt(@src(), "Param {d}", .{p.scratch.items.len - scratch_top});
+ defer trace_param.end();
+
+ if (p.currentTag() == .end_paren) break;
+
+ try p.scratch.append(p.gpa, try p.parseParamDecl());
+
+ switch (p.currentTag()) {
+ .comma => {
+ _ = p.nextToken();
+ },
+ .end_paren => break,
+ else => return p.fail(.{ .expected_token = .comma }),
+ }
+ }
+
+ _ = p.assertToken(.end_paren);
+
+ return p.scratchToSubList(scratch_top);
+}
+
+fn parseParamDecl(p: *Parse) !NodeIndex {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ const main_token = p.assertToken(.id);
+
+ _ = try p.expect(.colon);
+
+ const @"type" = try p.parseExpression();
+
+ return try p.addNode(.{
+ .tag = .param_decl,
+ .main_token = main_token,
+ .data = .{ .param_decl = .{
+ .type = @"type",
+ } },
+ });
+}
+
+fn parseBlock(p: *Parse) !NodeIndex {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ const main_token = p.assertToken(.start_brace);
+
+ const tag, const list = try p.parseBlockContents();
+ assert(p.lastTag() == .end_brace);
+
+ return try p.addNode(.{
+ .tag = tag,
+ .main_token = main_token,
+ .data = .{
+ .list = list,
+ },
+ });
+}
+
+fn parseLabelExpression(p: *Parse) !NodeIndex {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ const main_token = p.assertToken(.id);
+ _ = try p.expect(.colon);
+ const expr = try p.parseExpression();
+ return try p.addNode(.{
+ .tag = .label,
+ .main_token = main_token,
+ .data = .{ .label = .{
+ .expr = expr,
+ } },
+ });
+}
+
+fn parseBlockContents(p: *Parse) !struct { Node.Tag, SubList } {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ assert(p.lastTag() == .start_brace);
+ const scratch_top = p.scratch.items.len;
+ defer p.scratch.shrinkRetainingCapacity(scratch_top);
+
+ const tag: Node.Tag = while (true) switch (p.currentTag()) {
+ .kw_let,
+ .kw_var,
+ => try p.scratch.append(p.gpa, try p.parseVarDecl()),
+
+ .end_brace => break .block_void,
+
+ else => {
+ if (!p.isExpressionStart()) {
+ return p.fail(.expected_statement);
+ }
+
+ const first_tag = p.currentTag();
+
+ try p.scratch.append(p.gpa, try p.parseExpression());
+
+ switch (first_tag) {
+ // some control flow and blocks are allowed to not have a
+ // trailing semicolons. this is a stylistic choice.
+ .kw_if,
+ .start_brace,
+ => {
+ // ending a block here should also count as an implicit return,
+ // even though voiding it with a semicolon is not allowed.
+ if (p.currentTag() == .end_brace) break .block_implicit_return;
+ },
+
+ // TODO: make kw_break and kw_return REQUIRE a semicolon, as
+ // they are noreturn statements, doing implicit break on
+ // them makes no sense
+
+ else => {
+ if (p.eatToken(.semi) == null) {
+ // this must be last statement in block if it isnt,
+ // the p.expect(.end_brace) later will fail.
+ break .block_implicit_return;
+ }
+ },
+ }
+ },
+ };
+
+ _ = try p.expect(.end_brace);
+
+ return .{ tag, try p.scratchToSubList(scratch_top) };
+}
+
+/// VarDecl <- (kw_pub)? (kw_var | kw_let) id (colon expr)? equal expr semi
+fn parseVarDecl(p: *Parse) !NodeIndex {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ const kind = p.currentTag();
+ assert(kind == .kw_var or kind == .kw_let);
+
+ const main_token = p.nextToken();
+
+ _ = try p.expect(.id);
+
+ const data_type: OptionalNodeIndex = if (p.eatToken(.colon)) |_|
+ optionalNodeIndex(try p.parseExpression())
+ else
+ .none;
+
+ _ = try p.expect(.equal);
+
+ const data_init = try p.parseExpression();
+
+ _ = try p.expect(.semi);
+
+ return try p.addNode(.{
+ .tag = .var_decl,
+ .main_token = main_token,
+ .data = .{ .var_decl = .{
+ .type = data_type,
+ .init = data_init,
+ } },
+ });
+}
+
+fn isExpressionStart(p: *Parse) bool {
+ return switch (p.currentTag()) {
+ .id,
+ .number_int,
+ .number_float,
+ .string,
+ .builtin_id,
+ .kw_return,
+ .kw_break,
+ .kw_if,
+ .start_paren,
+ .start_brace,
+ .bang,
+ .tilde,
+ .minus,
+ => true,
+ else => false,
+ };
+}
+
+const Operator = enum(u8) {
+ add = @intFromEnum(Token.Tag.plus),
+ sub = @intFromEnum(Token.Tag.minus),
+ mul = @intFromEnum(Token.Tag.star),
+ div = @intFromEnum(Token.Tag.slash),
+ rem = @intFromEnum(Token.Tag.percent),
+ pow = @intFromEnum(Token.Tag.star_star),
+ @"and" = @intFromEnum(Token.Tag.and_and),
+ @"or" = @intFromEnum(Token.Tag.pipe_pipe),
+ bit_and = @intFromEnum(Token.Tag.@"and"),
+ bit_or = @intFromEnum(Token.Tag.pipe),
+ bit_xor = @intFromEnum(Token.Tag.caret),
+ shl = @intFromEnum(Token.Tag.lt_lt),
+ shr = @intFromEnum(Token.Tag.gt_gt),
+ eq = @intFromEnum(Token.Tag.equal_equal),
+ ne = @intFromEnum(Token.Tag.bang_equal),
+ lt = @intFromEnum(Token.Tag.lt),
+ le = @intFromEnum(Token.Tag.lt_equal),
+ gt = @intFromEnum(Token.Tag.gt),
+ ge = @intFromEnum(Token.Tag.gt_equal),
+ assign = @intFromEnum(Token.Tag.equal),
+ add_assign = @intFromEnum(Token.Tag.plus_equal),
+ sub_assign = @intFromEnum(Token.Tag.minus_equal),
+ mul_assign = @intFromEnum(Token.Tag.star_equal),
+ div_assign = @intFromEnum(Token.Tag.slash_equal),
+ rem_assign = @intFromEnum(Token.Tag.percent_equal),
+ pow_assign = @intFromEnum(Token.Tag.star_star_equal),
+ and_assign = @intFromEnum(Token.Tag.and_and_equal),
+ or_assign = @intFromEnum(Token.Tag.pipe_pipe_equal),
+ bit_and_assign = @intFromEnum(Token.Tag.and_equal),
+ bit_or_assign = @intFromEnum(Token.Tag.pipe_equal),
+ bit_xor_assign = @intFromEnum(Token.Tag.caret_equal),
+ shl_assign = @intFromEnum(Token.Tag.lt_lt_equal),
+ shr_assign = @intFromEnum(Token.Tag.gt_gt_equal),
+
+ const Direction = enum { ltr, rtl };
+
+ /// Only the binary operator levels are actually used,
+ /// but all are listed for reference.
+ const Level = enum {
+ /// e. e[] e()
+ accessor,
+ /// new
+ new,
+ /// ++e, --e, !e, ~e, +e, -e, typeof, void, delete
+ prefix,
+ /// **
+ exponentiation,
+ /// *, /, %
+ multiplicative,
+ /// +, -
+ additive,
+ /// <<, >>, >>>
+ bitwise_shift,
+ /// <, <=, >, >=
+ relational,
+ /// ==, !=
+ equality,
+ /// &
+ bit_and,
+ /// ^
+ bit_xor,
+ /// |
+ bit_or,
+ /// &&
+ logical_and,
+ /// ||, ??
+ logical_or,
+ /// =, +=, -=, *=, /=, %=, / <<=, >>=, >>>=, &=, ^=, |=
+ assign,
+ };
+
+ pub fn direction(op: Operator) Direction {
+ return switch (op) {
+ .pow => .rtl,
+
+ .assign,
+ .add_assign,
+ .sub_assign,
+ .mul_assign,
+ .div_assign,
+ .rem_assign,
+ .pow_assign,
+ .and_assign,
+ .or_assign,
+ .bit_and_assign,
+ .bit_or_assign,
+ .bit_xor_assign,
+ .shl_assign,
+ .shr_assign,
+ => .rtl,
+
+ else => .ltr,
+ };
+ }
+
+ pub fn level(op: Operator) Level {
+ return switch (op) {
+ .pow => .exponentiation,
+ .mul, .div, .rem => .multiplicative,
+ .add, .sub => .additive,
+ .shl, .shr => .bitwise_shift,
+ .lt, .le, .gt, .ge => .relational,
+ .eq, .ne => .equality,
+ .@"and" => .logical_and,
+ .@"or" => .logical_or,
+ .bit_and => .bit_and,
+ .bit_xor => .bit_xor,
+ .bit_or => .bit_or,
+ .assign,
+ .add_assign,
+ .sub_assign,
+ .mul_assign,
+ .div_assign,
+ .rem_assign,
+ .pow_assign,
+ .and_assign,
+ .or_assign,
+ .bit_and_assign,
+ .bit_or_assign,
+ .bit_xor_assign,
+ .shl_assign,
+ .shr_assign,
+ => .assign,
+ };
+ }
+
+ const lut = brk: {
+ const token_tags = std.enums.values(Token.Tag);
+ const op_tags = std.enums.values(Operator);
+
+ var is_operator_lut: [token_tags.len]bool = .{false} ** token_tags.len;
+ for (op_tags) |op| {
+ is_operator_lut[@intFromEnum(op)] = true;
+ }
+
+ var ast_tag_lut: [token_tags.len]Node.Tag = .{.root} ** token_tags.len;
+ for (op_tags) |op| {
+ ast_tag_lut[@intFromEnum(op)] = @field(Node.Tag, @tagName(op));
+ }
+
+ break :brk struct {
+ const is_operator = is_operator_lut;
+ const ast_tag = ast_tag_lut;
+ };
+ };
+
+ pub fn fromToken(tag: Token.Tag) ?Operator {
+ return if (lut.is_operator[@intFromEnum(tag)])
+ @enumFromInt(@intFromEnum(tag))
+ else
+ null;
+ }
+
+ pub fn astTag(op: Operator) Node.Tag {
+ const tag = lut.ast_tag[@intFromEnum(op)];
+ assert(tag != .root); // invalid operator or lut construction
+ return tag;
+ }
+};
+
+fn parseExpression(p: *Parse) anyerror!NodeIndex {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ assert(p.isExpressionStart());
+
+ // These unary operators are greedy, so handle them first.
+ switch (p.currentTag()) {
+ .kw_return => return p.parseReturn(),
+ .kw_break => return p.parseBreak(),
+ else => {},
+ }
+
+ var expr = try p.parseExpressionPart();
+
+ while (Operator.fromToken(p.currentTag())) |op| {
+ expr = try p.parseBinarySuffix(expr, op);
+ }
+
+ return expr;
+}
+
+fn parseExpressionPart(p: *Parse) !NodeIndex {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ var expr = switch (p.currentTag()) {
+ // Literals
+ .id => if (p.nextTag() == .colon)
+ try p.parseLabelExpression()
+ else
+ try p.blankNode(.id),
+ .number_int => try p.blankNode(.number_int),
+ .number_float => try p.blankNode(.number_float),
+ .string => try p.blankNode(.string),
+
+ // Complex expressions
+ .builtin_id => try p.parseBuiltinCall(),
+ .kw_return => try p.parseReturn(),
+ .kw_break => try p.parseBreak(),
+ .kw_if => try p.parseIf(),
+ .start_paren => try p.parseGroup(),
+ .start_brace => try p.parseBlock(),
+
+ // Unary operators
+ inline .bang, .tilde, .minus => |tag| unary: {
+ const main_token = p.nextToken();
+ const expr = try p.parseExpressionPart();
+
+ break :unary try p.addNode(.{
+ .tag = switch (tag) {
+ .bang => .bool_not,
+ .tilde => .bit_not,
+ .minus => .negate,
+ else => comptime unreachable,
+ },
+ .main_token = main_token,
+ .data = .{ .un = .{
+ .expr = expr,
+ } },
+ });
+ },
+
+ else => return p.fail(.expected_expr),
+ };
+
+ // parse field, calls, and index expressions
+ while (true) {
+ const trace2 = t.traceFmt(@src(), "(field/call)", .{});
+ defer trace2.end();
+
+ switch (p.currentTag()) {
+ .dot => {
+ const main_token = p.nextToken();
+ const field = try p.expect(.id);
+
+ expr = try p.addNode(.{
+ .tag = .field,
+ .main_token = main_token,
+ .data = .{ .field = .{
+ .base = expr,
+ .field_token = field,
+ } },
+ });
+ },
+ .start_paren => {
+ const main_token = p.nextToken();
+ const args = try p.parseCallArgs();
+
+ const extra = try p.addExtra(Ast.Call, .{
+ .args = args,
+ });
+
+ expr = try p.addNode(.{
+ .tag = .call,
+ .main_token = main_token,
+ .data = .{ .call = .{
+ .base = expr,
+ .extra = extra,
+ } },
+ });
+ },
+ .start_bracket => {
+ const main_token = p.nextToken();
+ const index = try p.parseExpression();
+
+ expr = try p.addNode(.{
+ .tag = .index,
+ .main_token = main_token,
+ .data = .{ .bin = .{
+ .lhs = expr,
+ .rhs = index,
+ } },
+ });
+ },
+
+ else => break,
+ }
+ }
+
+ return expr;
+}
+
+fn parseBinarySuffix(p: *Parse, lhs: NodeIndex, op: Operator) !NodeIndex {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ const main_token = p.nextToken();
+ const level = op.level();
+ var rhs = try p.parseExpressionPart();
+
+ while (Operator.fromToken(p.currentTag())) |next_op| {
+ const new_level = next_op.level();
+ const use = switch (op.direction()) {
+ .ltr => @intFromEnum(new_level) < @intFromEnum(level),
+ .rtl => @intFromEnum(new_level) <= @intFromEnum(level),
+ };
+ if (!use) break;
+
+ rhs = try p.parseBinarySuffix(rhs, next_op);
+ }
+
+ return try p.addNode(.{
+ .tag = op.astTag(),
+ .main_token = main_token,
+ .data = .{ .bin = .{
+ .lhs = lhs,
+ .rhs = rhs,
+ } },
+ });
+}
+
+fn parseReturn(p: *Parse) !NodeIndex {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ const main_token = p.assertToken(.kw_return);
+
+ const expr = if (p.isExpressionStart())
+ try p.parseExpression()
+ else
+ null;
+
+ return try p.addNode(.{
+ .tag = .@"return",
+ .main_token = main_token,
+ .data = .{ .@"return" = .{
+ .expr = optionalNodeIndex(expr),
+ } },
+ });
+}
+
+fn parseBreak(p: *Parse) !NodeIndex {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ const main_token = p.assertToken(.kw_break);
+
+ const label = if (p.eatToken(.colon)) |_|
+ try p.expect(.id)
+ else
+ null;
+
+ const expr = if (p.isExpressionStart())
+ try p.parseExpression()
+ else
+ null;
+
+ return try p.addNode(.{
+ .tag = .@"break",
+ .main_token = main_token,
+ .data = .{ .@"break" = .{
+ .label = optionalTokenIndex(label),
+ .value = optionalNodeIndex(expr),
+ } },
+ });
+}
+
+fn parseBuiltinCall(p: *Parse) !NodeIndex {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ const main_token = p.assertToken(.builtin_id);
+
+ _ = try p.expect(.start_paren);
+
+ const args = try p.parseCallArgs();
+
+ return try p.addNode(.{
+ .tag = .builtin_call,
+ .main_token = main_token,
+ .data = .{ .list = args },
+ });
+}
+
+fn parseCallArgs(p: *Parse) anyerror!SubList {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ const scratch_top = p.scratch.items.len;
+ defer p.scratch.shrinkRetainingCapacity(scratch_top);
+
+ while (p.isExpressionStart()) {
+ try p.scratch.append(p.gpa, try p.parseExpression());
+
+ switch (p.currentTag()) {
+ .end_paren => {
+ break;
+ },
+ .comma => {
+ _ = p.nextToken();
+ if (p.currentTag() == .end_paren)
+ break;
+ },
+ else => return p.fail(.{ .expected_token = .end_paren }),
+ }
+ }
+
+ _ = try p.expect(.end_paren);
+
+ return p.scratchToSubList(scratch_top);
+}
+
+fn parseIf(p: *Parse) !NodeIndex {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ const main_token = p.assertToken(.kw_if);
+
+ const cond = try p.parseExpression();
+
+ _ = try p.expectWithoutAdvance(.start_brace);
+ const yes = try p.parseBlock();
+
+ const no = if (p.eatToken(.kw_else)) |_|
+ try p.parseElseValue()
+ else
+ null;
+
+ const extra = try p.addExtra(Ast.If, .{
+ .yes = yes,
+ .no = optionalNodeIndex(no),
+ });
+
+ return try p.addNode(.{
+ .tag = .@"if",
+ .main_token = main_token,
+ .data = .{ .@"if" = .{
+ .cond = cond,
+ .extra = extra,
+ } },
+ });
+}
+
+fn parseElseValue(p: *Parse) anyerror!NodeIndex {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ assert(p.lastTag() == .kw_else);
+
+ return switch (p.currentTag()) {
+ .start_brace => p.parseBlock(),
+ .kw_if => p.parseIf(),
+ .id => p.parseLabelExpression(),
+
+ else => return p.fail(.expected_else_value),
+ };
+}
+
+fn parseGroup(p: *Parse) !NodeIndex {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ const main_token = p.assertToken(.start_paren);
+
+ const expr = try p.parseExpression();
+
+ _ = try p.expect(.end_paren);
+
+ return try p.addNode(.{
+ .tag = .group,
+ .main_token = main_token,
+ .data = .{ .un = .{
+ .expr = expr,
+ } },
+ });
+}
+
+// helpers
+
+/// Append a node and return it's index.
+fn addNode(p: *Parse, node: Node) !NodeIndex {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ try p.nodes.append(p.gpa, node);
+ // @intCast is always safe because 2^32 tokens alwayss produce less than
+ // 2^32 nodes, and there is a 2^32 limit on source file bytes.
+ return @intCast(p.nodes.len - 1);
+}
+
+fn addExtra(p: *Parse, comptime T: type, extra: T) !ExtraIndex {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ const extra_top = p.extra.items.len;
+ const bytes = std.mem.asBytes(&extra);
+ try p.extra.appendSlice(p.gpa, std.mem.bytesAsSlice(u32, bytes));
+ return @intCast(extra_top);
+}
+
+fn fail(p: *Parse, ast_error: AstError) Error {
+ p.ast_error = ast_error;
+ std.debug.print("got {s}", .{@tagName(p.currentTag())});
+ return Error.SyntaxError;
+}
+
+fn blankNode(p: *Parse, comptime tag: Token.Tag) !NodeIndex {
+ const main_token = p.assertToken(tag);
+ return try p.addNode(.{
+ .tag = @field(Node.Tag, @tagName(tag)),
+ .main_token = main_token,
+ .data = .{ .none = .{} },
+ });
+}
+
+fn currentTag(p: *Parse) Token.Tag {
+ return p.token_tags[p.pos];
+}
+
+fn lastTag(p: *Parse) Token.Tag {
+ return p.token_tags[p.pos - 1];
+}
+
+fn nextTag(p: *Parse) Token.Tag {
+ return p.token_tags[p.pos + 1];
+}
+
+fn eatToken(p: *Parse, tag: Token.Tag) ?TokenIndex {
+ return if (p.token_tags[p.pos] == tag) p.nextToken() else null;
+}
+
+fn expect(p: *Parse, tag: Token.Tag) !TokenIndex {
+ if (p.currentTag() != tag) {
+ return p.fail(.{ .expected_token = tag });
+ }
+ return p.nextToken();
+}
+
+fn expectWithoutAdvance(p: *Parse, tag: Token.Tag) !void {
+ if (p.currentTag() != tag) {
+ return p.fail(.{ .expected_token = tag });
+ }
+}
+
+fn assertToken(p: *Parse, tag: Token.Tag) TokenIndex {
+ const token = p.nextToken();
+ assert(p.token_tags[token] == tag);
+ return token;
+}
+
+fn nextToken(p: *Parse) TokenIndex {
+ const result = p.pos;
+ p.pos += 1;
+ return result;
+}
+
+fn tokensOnSameLine(p: *Parse, token1: TokenIndex, token2: TokenIndex) bool {
+ return std.mem.indexOfScalar(u8, p.source[p.token_starts[token1]..p.token_starts[token2]], '\n') == null;
+}
+
+fn fullToken(p: *Parse, token: TokenIndex) Token {
+ return Token{
+ .tag = p.token_tags[token],
+ .offset = p.token_starts[token],
+ };
+}
+
+fn scratchToSubList(p: *Parse, scratch_top: usize) !SubList {
+ const trace = t.trace(@src());
+ defer trace.end();
+
+ const items = p.scratch.items[scratch_top..];
+ const extra_top = p.extra.items.len;
+ try p.extra.appendSlice(p.gpa, p.scratch.items[scratch_top..]);
+ return .{
+ .start = @intCast(extra_top),
+ .len = @intCast(items.len),
+ };
+}
+
+const Parse = @This();
+
+const std = @import("std");
+const assert = std.debug.assert;
+const mem = std.mem;
+const Allocator = mem.Allocator;
+const ArrayListUnmanaged = std.ArrayListUnmanaged;
+
+const Tokenizer = @import("Tokenizer.zig");
+const Token = Tokenizer.Token;
+const ByteOffset = Tokenizer.ByteOffset;
+
+const Ast = @import("Ast.zig");
+const Node = Ast.Node;
+const Data = Node.Data;
+const SubList = Ast.SubList;
+const NodeIndex = Ast.NodeIndex;
+const TokenIndex = Ast.TokenIndex;
+const ExtraIndex = Ast.ExtraIndex;
+const OptionalNodeIndex = Ast.OptionalNodeIndex;
+const OptionalTokenIndex = Ast.OptionalTokenIndex;
+const OptionalExtraIndex = Ast.OptionalExtraIndex;
+const optionalTokenIndex = OptionalTokenIndex.init;
+const optionalNodeIndex = OptionalNodeIndex.init;
+const optionalExtraIndex = OptionalExtraIndex.init;
+
+const t = @import("tracer");
diff --git a/src/compiler/Report.zig b/src/compiler/Report.zig
new file mode 100644
index 0000000000000000000000000000000000000000..53f1509e0027bdb5e6590dfb36fb9858b5a7e6a4
--- /dev/null
+++ b/src/compiler/Report.zig
@@ -0,0 +1,38 @@
+errors: []Item = &.{},
+warns: []Item = &.{},
+string_bytes: []u8 = &.{},
+
+pub const Item = struct {
+ offset: ByteOffset,
+ message: String,
+};
+
+pub const String = struct {
+ start: u32,
+ len: u32,
+
+ pub fn message(item: Item, report: Report) []const u8 {
+ return report.string_bytes[item.message_start..][0..item.message_len];
+ }
+};
+
+pub fn hasErrors(report: Report) bool {
+ return report.errors.len > 0;
+}
+
+pub fn hasWarns(report: Report) bool {
+ return report.warns.len > 0;
+}
+
+pub const Wip = struct {
+ errors: ArrayListUnmanaged(Item),
+ warns: ArrayListUnmanaged(Item),
+ string_bytes: ArrayListUnmanaged(u8),
+};
+
+const Report = @This();
+const std = @import("std");
+const ArrayListUnmanaged = std.ArrayListUnmanaged;
+const assert = std.debug.assert;
+pub const Tokenizer = @import("Tokenizer.zig");
+pub const ByteOffset = Tokenizer.ByteOffset;
diff --git a/src/compiler/Sema.zig b/src/compiler/Sema.zig
new file mode 100644
index 0000000000000000000000000000000000000000..5dd1f1e735811f7f247eaf748a5a0cb8c6875b69
--- /dev/null
+++ b/src/compiler/Sema.zig
@@ -0,0 +1,693 @@
+/// absolutely stealing zig on this one haha
+/// This struct is created per declaration and chills
+
+// inputs:
+comp: *Comp,
+gpa: Allocator,
+arena: Allocator,
+code: Teg,
+ip: *InternPool,
+decl: *Decl,
+decl_index: Decl.Index,
+namespace_index: Namespace.Index,
+
+// temp data:
+inst_map: InstMap = undefined,
+/// scratch memory. can be used in any place you might temporarily allocate an arraylist
+/// this is most notably used to store block contents before the block is finalized
+scratch: std.ArrayListUnmanaged(u32) = .{},
+params: []InternPool.Index = undefined,
+
+// outputs
+air_instructions: std.MultiArrayList(Air.Inst) = .{},
+air_extra: dod.ExtraStore(.air, .@"var") = .{},
+
+pub const Error = error{
+ /// Analysis has failed, see error list
+ AnalysisFail,
+ /// This analysis is waiting on other dependencies to finish
+ Interrupt,
+};
+
+/// Instead of using a hash map for this, just pre-allocate a list long enough to hold everything.
+/// This is fine because all instructions are going to be adacent in a single declaration.
+const InstMap = struct {
+ items: []Air.Inst.Ref.Optional,
+ start: Teg.Inst.Index,
+
+ pub fn init(allocator: Allocator, code: *const Teg, inst_start: Teg.Inst.Index) !InstMap {
+ const decl = code.declData(inst_start);
+ log.info("pre-allocating {d} map-indicies for InstMap", .{decl.instruction_count});
+ const insts = try allocator.alloc(Air.Inst.Ref.Optional, decl.instruction_count);
+ @memset(insts, .none);
+ return .{
+ .items = insts,
+ .start = inst_start,
+ };
+ }
+
+ pub fn deinit(map: *InstMap, allocator: Allocator) void {
+ allocator.free(map.items);
+ map.* = undefined;
+ }
+
+ pub fn get(map: *InstMap, teg_code: Teg.Inst.Index) ?Air.Inst.Ref {
+ const source_i = @intFromEnum(teg_code);
+ const start_i = @intFromEnum(map.start);
+ assert(source_i >= start_i and source_i <= (start_i + map.items.len));
+ return map.items[source_i - start_i].unwrap();
+ }
+
+ pub fn add(map: *InstMap, from: Teg.Inst.Index, to: Air.Inst.Ref) !void {
+ assert(map.get(from) == null);
+ const source_i = @intFromEnum(from);
+ const start_i = @intFromEnum(map.start);
+ map.items[source_i - start_i] = to.toOptional();
+ }
+};
+
+pub fn init(s: *Sema) !void {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ try s.comp.current_compile_decls.append(s.gpa, s.decl_index);
+
+ log.info("init sema on index {}", .{s.decl_index});
+ s.inst_map = try InstMap.init(
+ s.arena,
+ &s.code,
+ s.ip.declPtr(s.decl_index).teg_decl_index.unwrap().?.resolve(s.ip),
+ );
+}
+
+pub fn deinit(s: *Sema) void {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ defer s.* = undefined;
+ s.inst_map.deinit(s.arena);
+ s.scratch.deinit(s.arena);
+ s.air_extra.deinit(s.gpa);
+ s.air_instructions.deinit(s.gpa);
+}
+
+pub fn finalize(s: *Sema) !void {
+ const air: Air = s.tempAir();
+
+ std.debug.print("=" ** 20, .{});
+ std.debug.print("\nAir for @ {d}\n", .{@intFromEnum(s.decl.teg_decl_index)});
+ air.dumpToWriter(s.ip, std.io.getStdErr().writer()) catch {};
+
+ try s.comp.codeGenDecl(s.decl, &air);
+}
+
+// --- analyze functions ---
+
+/// This is the entrypoint to semantic analysis: It all starts at a declaration.
+pub fn analyzeDeclaration(s: *Sema) anyerror!Value {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ // TODO: turn this entire function into a nonrecursive state machine, so
+ // that analysis can pause and continue at any point. for example, when a
+ // dependency on another declaration is required, but that declaration has
+ // not been resolved yet. or, we are waiting on a file read to happen
+
+ const decl, const fn_decl = s.code.fullFuncDecl(s.inst_map.start);
+ assert(decl.kind == .function);
+
+ s.params = try s.gpa.alloc(InternPool.Index, fn_decl.params.len);
+ defer s.gpa.free(s.params);
+
+ for (fn_decl.params.slice(&s.code), 0..) |param, i| {
+ s.params[i] = (try s.resolveRefComptime(param)).toIntern();
+ }
+
+ const return_type = try s.resolveRefComptime(fn_decl.return_type);
+
+ const root = (try s.analyzeBlock(
+ fn_decl.body.toIndex() orelse
+ @panic("TODO: please change this to Index. Ref is not allowed here anymore"),
+ )).unwrap().inst;
+ assert(@intFromEnum(root) == 0);
+
+ const func_prototype = try s.ip.get(s.gpa, .{
+ .type_function = .{
+ .param_types = s.params,
+ .return_type = return_type.toIntern(),
+ },
+ });
+ const func_value = try s.ip.get(s.gpa, .{
+ .func = .{
+ .ty = func_prototype,
+ .owner_decl = s.decl_index,
+ .teg_body_inst = s.decl.teg_decl_index.unwrap().?,
+ },
+ });
+ const value = Value.fromInterned(func_value);
+ s.decl.val = value;
+ return value;
+}
+
+pub fn analyzeBlock(s: *Sema, in: Teg.Inst.Index) anyerror!Air.Inst.Ref {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ const block = try s.addBlock();
+
+ try s.inst_map.add(in, block.block_index.toRef());
+
+ const scratch_top = s.scratch.items.len;
+ defer s.scratch.shrinkRetainingCapacity(scratch_top);
+
+ var last: Air.Inst.Ref = undefined;
+ const instructions = s.code.blockInstructions(in);
+ assert(instructions.len != 0);
+ for (instructions) |inst| {
+ last = try s.analyzeCode(inst);
+ }
+
+ if (scratch_top + 1 == s.scratch.items.len) {
+ const single_instruction = s.air_instructions.items(.tag)[@intFromEnum(block.block_index) + 1];
+ if (single_instruction == .@"break") {
+ const data: Air.Break = s.air_instructions.items(.data)[@intFromEnum(block.block_index) + 1]
+ .decode(.@"break", single_instruction);
+ if (data.block_ref == block.block_index) {
+ // This is a block that resolved to just a single value ref with
+ // no side effects. This means we can just lower this to the ref
+ // itself, and give codegen an easier time.
+
+ // Delete the block and the single instruction
+ // Break does not use any extra data, so this call removes both items
+ s.air_instructions.shrinkRetainingCapacity(@intFromEnum(block.block_index));
+
+ return data.value;
+ }
+ }
+ }
+
+ return (try block.finish(s, scratch_top)).toRef();
+}
+
+fn analyzeCode(s: *Sema, in: Teg.Inst.Index) !Air.Inst.Ref {
+ const t = tracer.traceFmt(@src(), "%{d}, {s}", .{ @intFromEnum(in), @tagName(s.code.instTag(in)) });
+ defer t.end();
+
+ log.info("runtime %{d} = .{s}", .{ @intFromEnum(in), @tagName(s.code.instTag(in)) });
+
+ const out = try s.analyzeCodeInner(in);
+ if (s.code.instTag(in) != .block)
+ try s.inst_map.add(in, out);
+
+ return out;
+}
+
+fn analyzeCodeInner(s: *Sema, in: Teg.Inst.Index) !Air.Inst.Ref {
+ switch (s.code.instTag(in)) {
+ .string => {
+ const string: Teg.SlicedString = s.code.instPayload(.string, in);
+ return s.makeValueAir(.{ .string = string.slice(s.code.string_bytes) });
+ },
+ .number => {
+ const number: dod.Float = s.code.instPayload(.number, in);
+ return s.makeValueAir(.{ .float = number.decode() });
+ },
+ .js => {
+ const operand = try s.resolveRefComptime(s.code.instData(.js, in).operand);
+ try s.validateType(operand.type(s.ip), .string_type);
+ return s.addInstRef(.js_global_ref, operand.asString(s.ip));
+ },
+ .field => {
+ const field: Teg.Inst.Bin = s.code.instPayload(.field, in);
+ const base = try s.resolveRefRuntime(field.lhs);
+ const rhs = try s.resolveRefComptime(field.rhs);
+
+ try s.validateType(rhs.type(s.ip), .string_type);
+ const string = rhs.asString(s.ip);
+
+ const base_type = s.typeOfRef(base);
+ log.info("process field with type {any} . {s}", .{ base_type, string.toSlice(s.ip) });
+
+ switch (base_type) {
+ .opaque_type => {
+ return s.addInstRef(.field, .{
+ .base = base,
+ .field = try string.tryToNullTerminatedString(s.ip, s.gpa),
+ });
+ },
+ else => |tag| std.debug.panic("TODO: field analyze with base .{s}", .{@tagName(tag)}),
+ }
+ },
+ .load_decl => {
+ const request: Teg.Inst.StrNode = s.code.instData(.load_decl, in);
+ const decl_name = try s.ip.getOrPutString(
+ s.gpa,
+ request.string.slice(s.code.string_bytes),
+ .no_embedded_nulls,
+ );
+ const decl_index = s.lookupDecl(decl_name) orelse {
+ std.debug.print("did not find {s}\n", .{decl_name.toSlice(s.ip)});
+ return error.AnalysisFail;
+ };
+ const resolved = try s.resolveDecl(decl_index);
+ return resolved.toIntern().toAir();
+ },
+ .param => {
+ const param: Teg.Inst.DataNode = s.code.instData(.param, in);
+ return s.addInstRef(.arg, .{
+ .type = Value.fromInterned(s.params[param.data]),
+ .arg_number = param.data,
+ });
+ },
+ inline .add,
+ .sub,
+ .mul,
+ .div,
+ .lt,
+ .gt,
+ .eq,
+ => |tag| {
+ const bin: Teg.Inst.Bin = s.code.instPayload(tag, in);
+
+ const lhs = try s.resolveRefRuntime(bin.lhs);
+ const rhs = try s.resolveRefRuntime(bin.rhs);
+
+ if (lhs.kind == .value and rhs.kind == .value) {
+ @panic("TODO: comptime math operation");
+ }
+
+ const left_type = s.typeOfRef(lhs);
+ const right_type = s.typeOfRef(rhs);
+
+ // TODO: validate type for operation
+
+ log.info("runtime {s} operation on {} and {}", .{
+ @tagName(tag),
+ left_type.toKey(s.ip),
+ right_type.toKey(s.ip),
+ });
+
+ // todo: type check
+
+ return s.addInstRef(tag.airTag(), .{
+ .lhs = lhs,
+ .rhs = rhs,
+ });
+ },
+ inline .negate => |tag| {
+ const un: Teg.Inst.RefNode = s.code.instData(tag, in);
+
+ const operand = try s.resolveRefRuntime(un.operand);
+
+ const operand_type = s.typeOfRef(operand);
+
+ // TODO: validate type for operation
+
+ log.info("runtime {s} operation on {}", .{
+ @tagName(tag),
+ operand_type.toKey(s.ip),
+ });
+
+ return s.addInstRef(tag.airTag(), .{
+ .operand = operand,
+ });
+ },
+ .@"break" => {
+ const brk: Teg.Inst.Break = s.code.instPayload(.@"break", in);
+ const operand = try s.resolveRefRuntime(brk.operand);
+ const block_ref = s.inst_map.get(brk.block).?.unwrap().inst; // block ref is resolved by teg
+ return s.addInstRef(.@"break", .{
+ .block_ref = block_ref,
+ .value = operand,
+ });
+ },
+ .call => {
+ const call: Teg.Inst.Call = s.code.instPayload(.call, in);
+ const base = try s.resolveRefRuntime(call.base);
+ const base_type = s.typeOfRef(base).toKey(s.ip);
+
+ const is_opaque = base_type == .simple_type and base_type.simple_type == .@"opaque";
+
+ const scratch = try s.arena.alloc(Air.Inst.Ref, call.args.len);
+ defer s.arena.free(scratch);
+ if (!is_opaque) {
+ if (base_type != .type_function) {
+ std.debug.print("Attempt to call non-function {}", .{base_type});
+ return error.AnalysisFail;
+ }
+
+ const proto = base_type.type_function;
+ if (proto.param_types.len != call.args.len) {
+ std.debug.print("Incorrect args. Expect {d}, Got {d}", .{
+ proto.param_types.len,
+ call.args.len,
+ });
+ return error.AnalysisFail;
+ }
+
+ // validate function args
+ for (proto.param_types, call.args.slice(&s.code), 0..) |param, arg, i| {
+ const ref = try s.resolveRefRuntime(arg);
+ scratch[i] = ref;
+ try s.validateType(
+ Value.fromInterned(s.typeOfRef(ref)),
+ param,
+ );
+ }
+ } else {
+ // just copy the args in without checking
+ for (call.args.slice(&s.code), 0..) |arg, i| {
+ const ref = try s.resolveRefRuntime(arg);
+ scratch[i] = ref;
+ }
+ }
+
+ const start = s.air_extra.list.items.len;
+ try s.air_extra.list.appendSlice(s.gpa, @ptrCast(scratch));
+
+ return s.addInstRef(.call, .{
+ .base = base,
+ .call_data = try s.air_extra.append(s.gpa, Air.Call.Data, .{
+ .args = .{
+ .start = @intCast(start),
+ .len = @intCast(scratch.len),
+ },
+ }),
+ });
+ },
+ .ensure_void => {
+ const operand = try s.resolveRefRuntimeExtra(s.code.instData(.ensure_void, in).operand, false);
+ const t = Value.fromInterned(s.typeOfRef(operand));
+ if (!t.isEnsureVoid(s.ip)) {
+ std.debug.print("oh no, we are discarding {} on accident\n", .{
+ t.fmt(s.ip),
+ });
+ }
+ return InternPool.Index.void_value.toAir();
+ },
+ .@"if" => {
+ const data: Teg.Inst.If = s.code.instData(.@"if", in).extra_index.get(&s.code);
+
+ const cond = try s.resolveRefRuntime(data.cond);
+
+ try s.validateType(Value.fromInterned(s.typeOfRef(cond)), .bool_type);
+
+ // If known at comptime, only analyze the taken path
+ if (cond.kind == .value) {
+ @panic("TODO: comptime IF");
+ }
+
+ // Emit a runtime branch
+ const yes_block = try s.analyzeRefToBlock(data.yes);
+ const no_block = try s.analyzeRefToBlock(data.no);
+
+ return try s.addInstRef(.@"if", .{
+ .cond = cond,
+ .data = try s.air_extra.append(s.gpa, Air.If.Data, .{
+ .yes = yes_block,
+ .no = no_block,
+ .union_ty = s.unionType(Value.fromInterned(s.typeOfRef(yes_block)), Value.fromInterned(s.typeOfRef(no_block))),
+ }),
+ });
+ },
+ .block => {
+ const block = try s.analyzeBlock(in);
+ switch (block.unwrap()) {
+ .inst => |t| try s.scratch.append(s.gpa, @intFromEnum(t)),
+ else => {},
+ }
+ return block;
+ },
+ else => |tag| std.debug.panic("TODO: analyze of .{s}", .{@tagName(tag)}),
+ }
+}
+
+fn markInst(s: *Sema, inst: Air.Inst.Index) void {
+ const flags: *Air.Flags = &s.air_instructions.items(.flags)[@intFromEnum(inst)];
+ flags.uses = flags.uses.increment();
+ log.info("markInst {}", .{inst});
+}
+
+fn analyzeRefToBlock(s: *Sema, in: Teg.Inst.Ref) anyerror!Air.Inst.Ref {
+ const t = tracer.trace(@src());
+ defer t.end();
+
+ const scratch_top = s.scratch.items.len;
+ defer s.scratch.shrinkRetainingCapacity(scratch_top);
+
+ const visited = try s.resolveRefRuntime(in);
+
+ if (scratch_top == s.scratch.items.len) {
+ return visited;
+ }
+
+ if (visited.kind == .inst and
+ s.air_instructions.items(.tag)[@intFromEnum(visited.unwrap().inst)] == .block)
+ {
+ return visited;
+ }
+
+ const block = try s.addBlock();
+ return (try block.finish(s, scratch_top)).toRef();
+}
+
+fn resolveRefRuntimeExtra(s: *Sema, ref: Teg.Inst.Ref, mark: bool) anyerror!Air.Inst.Ref {
+ const index = ref.toIndex() orelse {
+ return ref.toInterned().?.toAir();
+ };
+
+ if (s.inst_map.get(index)) |air| {
+ if (mark) {
+ switch (air.unwrap()) {
+ .inst => |i| s.markInst(i),
+ else => {},
+ }
+ }
+ return air;
+ }
+
+ return s.analyzeCode(index);
+}
+
+fn resolveRefRuntime(s: *Sema, ref: Teg.Inst.Ref) anyerror!Air.Inst.Ref {
+ return resolveRefRuntimeExtra(s, ref, true);
+}
+
+fn resolveRefComptime(s: *Sema, ref: Teg.Inst.Ref) !Value {
+ const in = ref.toIndex() orelse {
+ return Value{ .ip_index = ref.toInterned().? };
+ };
+
+ if (s.inst_map.get(in)) |air| {
+ switch (air.unwrap()) {
+ .value => |value| return value,
+ else => {},
+ }
+ }
+
+ const tag = s.code.instTag(in);
+ log.info("comptime %{d} = .{s}", .{ @intFromEnum(in), @tagName(tag) });
+ switch (tag) {
+ .string => {
+ const string: Teg.SlicedString = s.code.instPayload(.string, in);
+ return s.makeValue(.{ .string = string.slice(s.code.string_bytes) });
+ },
+
+ else => std.debug.panic("TODO: comptime analyze of .{s}", .{@tagName(tag)}),
+ }
+}
+
+fn resolveDecl(s: *Sema, decl_index: Decl.Index) !Value {
+ const decl = s.ip.declPtr(decl_index);
+ switch (decl.analysis) {
+ .unreferenced => {
+ decl.analysis = .in_progress;
+
+ // TODO: make this short-circuit the current analysis, move this analyze
+ // job into a "deferred" list, and then make this thread analyze something else
+
+ var arena = std.heap.ArenaAllocator.init(s.gpa);
+ defer arena.deinit();
+
+ var sema: Sema = .{
+ .comp = s.comp,
+ .gpa = s.gpa,
+ .ip = s.ip,
+ .arena = arena.allocator(),
+ .code = try s.ip.namespacePtr(decl.src_namespace).file_scope.getTeg(s.gpa),
+ .decl = decl,
+ .decl_index = decl_index,
+ .namespace_index = decl.src_namespace,
+ };
+
+ try sema.init();
+ defer sema.deinit();
+ const value = try sema.analyzeDeclaration();
+ try sema.finalize();
+
+ decl.analysis = .complete;
+
+ return value;
+ },
+ .complete => {
+ return decl.val;
+ },
+ else => |tag| std.debug.panic("TODO: load_decl when decl is in state .{s}", .{@tagName(tag)}),
+ }
+}
+
+fn unionType(s: *Sema, a: Value, b: Value) Value {
+ if (std.debug.runtime_safety) {
+ assert(a.type(s.ip).toIntern() == .type_type);
+ assert(b.type(s.ip).toIntern() == .type_type);
+ }
+
+ if (a.ip_index == b.ip_index) {
+ return a;
+ }
+
+ if (a.ip_index == .noreturn_type)
+ return b;
+
+ if (b.ip_index == .noreturn_type)
+ return a;
+
+ std.debug.panic(
+ "TODO: when two non-equal types implicitly merge (if/match), this should generate a union type.\na: {}\nb: {}",
+ .{
+ a.fmt(s.ip),
+ b.fmt(s.ip),
+ },
+ );
+ // return s.ip.get(s.gpa, .{ .@"union" = &.{ a, b } });
+}
+
+/// --- helper ---
+const IncompleteBlock = struct {
+ block_index: Air.Inst.Index,
+
+ pub fn finish(block: IncompleteBlock, s: *Sema, scratch_top: usize) !Air.Inst.Index {
+ s.setInstData(.block, block.block_index, try s.scratchToIndexList(scratch_top));
+ return block.block_index;
+ }
+};
+
+fn addBlock(s: *Sema) !IncompleteBlock {
+ log.info("start block at @{d}", .{s.air_instructions.len});
+ try s.air_instructions.append(s.gpa, .{
+ .tag = .block,
+ .data = undefined,
+ .flags = .{},
+ });
+ return .{ .block_index = @enumFromInt(s.air_instructions.len - 1) };
+}
+
+fn addInstRef(s: *Sema, comptime tag: Air.Inst.Tag, data: tag.Data()) !Air.Inst.Ref {
+ log.info("add air .{s}, {}", .{ @tagName(tag), data });
+ try s.air_instructions.append(s.gpa, .{
+ .tag = tag,
+ .data = Air.EncodedData.encode(tag, data),
+ .flags = .{},
+ });
+ const index: Air.Inst.Index = @enumFromInt(s.air_instructions.len - 1);
+ try s.scratch.append(s.arena, @intFromEnum(index));
+ return index.toRef();
+}
+
+fn setInstData(
+ s: *Sema,
+ comptime tag: Air.Inst.Tag,
+ index: Air.Inst.Index,
+ data: tag.Data(),
+) void {
+ const i = @intFromEnum(index);
+ assert(s.air_instructions.items(.tag)[i] == tag);
+ s.air_instructions.items(.data)[i] = Air.EncodedData.encode(tag, data);
+}
+
+fn makeValue(s: *Sema, key: InternPool.Key) !Value {
+ return .{ .ip_index = try s.ip.get(s.gpa, key) };
+}
+
+fn makeValueAir(s: *Sema, key: InternPool.Key) !Air.Inst.Ref {
+ return (try s.makeValue(key)).ip_index.toAir();
+}
+
+fn mapTeg(s: *Sema, inst: Teg.Inst.Ref) Air.Inst.Ref {
+ if (inst.toInterned()) |interned| {
+ return interned.toAir();
+ }
+ const index = inst.toIndex().?;
+ return s.inst_map.get(index) orelse {
+ if (std.debug.runtime_safety) {
+ std.debug.panic("Unmapped operand %{d}", .{@intFromEnum(index)});
+ }
+ unreachable;
+ };
+}
+
+fn validateType(s: *Sema, checked_type: Value, wanted_type: InternPool.Index) !void {
+ if (checked_type.toIntern() != wanted_type) {
+ log.info("failed type check {}, got {}", .{ Value.fromInterned(wanted_type).fmt(s.ip), checked_type.fmt(s.ip) });
+ return error.TypeCheckFailed;
+ } else {
+ log.info("passed type check {}", .{checked_type.fmt(s.ip)});
+ }
+}
+
+fn scratchToIndexList(s: *Sema, scratch_top: usize) !Air.IndexList {
+ const items = s.scratch.items[scratch_top..];
+ const extra_top = s.air_extra.list.items.len;
+ try s.air_extra.list.appendSlice(s.gpa, s.scratch.items[scratch_top..]);
+ return .{
+ .start = @intCast(extra_top),
+ .len = @intCast(items.len),
+ };
+}
+
+fn tempAir(s: *const Sema) Air {
+ return .{
+ .insts = s.air_instructions.slice(),
+ .extra = s.air_extra.constant(),
+ };
+}
+
+fn typeOfRef(s: *Sema, ref: Air.Inst.Ref) InternPool.Index {
+ return s.tempAir().typeOf(ref, s.ip);
+}
+
+fn lookupDecl(s: *Sema, name: NullTerminatedString) ?InternPool.DeclIndex {
+ var nsi = s.namespace_index;
+ while (true) {
+ const ns = s.ip.namespacePtr(nsi);
+ if (ns.lookupDirect(s.comp, name)) |decl_id| {
+ return decl_id;
+ }
+ nsi = ns.parent.unwrap() orelse
+ unreachable; // transform ensures all identifiers get resolved
+ }
+}
+
+const Sema = @This();
+
+const std = @import("std");
+const mem = std.mem;
+const Allocator = std.mem.Allocator;
+const assert = std.debug.assert;
+const log = std.log.scoped(.sema);
+
+const Comp = @import("Comp.zig");
+const File = Comp.File;
+const Decl = Comp.Decl;
+const Namespace = Comp.Namespace;
+
+const Air = @import("Air.zig");
+
+const Teg = @import("Teg.zig");
+
+const Value = @import("Value.zig");
+const InternPool = @import("InternPool.zig");
+const NullTerminatedString = InternPool.NullTerminatedString;
+const dod = @import("dod.zig");
+
+const tracer = @import("tracer");
diff --git a/src/compiler/Teg.Dump.zig b/src/compiler/Teg.Dump.zig
new file mode 100644
index 0000000000000000000000000000000000000000..172ca4e9274ffb754d658e5a0ea3dfd638ab4991
--- /dev/null
+++ b/src/compiler/Teg.Dump.zig
@@ -0,0 +1,292 @@
+//! This implements a debug formatter for Teg
+teg: Teg,
+
+const State = struct {
+ arena: std.mem.Allocator,
+ teg: *const Teg,
+ indent: usize = 0,
+ tty: std.io.tty.Config,
+ visited: std.AutoHashMapUnmanaged(Teg.Inst.Index, void) = .{},
+
+ fn newline(state: State, writer: anytype) !void {
+ try writer.writeByte('\n');
+ if (state.indent > 0)
+ try writer.writeByteNTimes(' ', state.indent * 2);
+ }
+
+ fn dumpInst(s: *State, writer: anytype, inst: Teg.Inst.Index) @TypeOf(writer).Error!void {
+ const teg = s.teg;
+ const tty = s.tty;
+
+ try writer.print("%{} = ", .{@intFromEnum(inst)});
+
+ const tag = s.teg.instTag(inst);
+
+ try tty.setColor(writer, .cyan);
+ try writer.print("{s}", .{@tagName(tag)});
+ try tty.setColor(writer, .reset);
+
+ if ((try s.visited.fetchPut(s.arena, inst, {})) != null) {
+ try tty.setColor(writer, .red);
+ try writer.writeAll(" [error: recursive tree]");
+ try tty.setColor(writer, .reset);
+ return;
+ }
+
+ switch (tag) {
+ .module => {
+ try s.dumpInstList(writer, teg.moduleDeclarations(inst), '{', '}');
+ },
+ .declaration => {
+ const decl_info = teg.declData(inst);
+ try writer.print(" ({s}) ", .{@tagName(decl_info.kind)});
+
+ const name = decl_info.name.slice(teg.string_bytes);
+ if (name.len == 0) {
+ try tty.setColor(writer, .bold);
+ try writer.writeAll("");
+ } else {
+ try tty.setColor(writer, .magenta);
+ try writer.writeAll(name);
+ }
+ try tty.setColor(writer, .reset);
+
+ try writer.writeAll(" {");
+ s.indent += 1;
+ try s.newline(writer);
+
+ switch (decl_info.kind) {
+ .function => {
+ const fn_data: Teg.Inst.DeclFn = teg.fullFuncDecl(inst).@"1";
+ try writer.writeAll("params: ");
+ try s.dumpRefList(writer, fn_data.params.slice(teg), '[', ']');
+ try s.newline(writer);
+ try writer.writeAll("return_type: ");
+ try s.dumpRef(writer, fn_data.return_type);
+ try s.newline(writer);
+ try writer.writeAll("body: ");
+ try s.dumpRef(writer, fn_data.body);
+ },
+ }
+ s.indent -= 1;
+ try s.newline(writer);
+ try writer.writeAll("}");
+ },
+ .param => {
+ const data = teg.instData(.param, inst);
+ try writer.writeAll("(");
+ try tty.setColor(writer, .yellow);
+ try writer.print("{d}", .{data.data});
+ try tty.setColor(writer, .reset);
+ try writer.writeAll(")");
+ },
+ .block => {
+ const block = teg.instData(.block, inst);
+ const index_list = block.extra_index.get(teg);
+ try s.dumpInstList(writer, index_list.slice(teg), '[', ']');
+ },
+ .number => {
+ const number = teg.instData(.number, inst);
+ const float = number.extra_index.get(teg).decode();
+ try tty.setColor(writer, .yellow);
+ try writer.print(" {d}", .{float});
+ try tty.setColor(writer, .reset);
+ },
+ inline .field,
+ .store,
+ .add,
+ .sub,
+ .mul,
+ .div,
+ .rem,
+ .pow,
+ .@"and",
+ .@"or",
+ .bit_and,
+ .bit_or,
+ .bit_xor,
+ .shl,
+ .shr,
+ .eq,
+ .ne,
+ .lt,
+ .le,
+ .gt,
+ .ge,
+ => |comptime_tag| {
+ const data = teg.instData(comptime_tag, inst).extra_index.get(teg);
+ try writer.writeAll("(");
+ try s.dumpRefMarker(writer, data.lhs);
+ try writer.writeAll(", ");
+ try s.dumpRefMarker(writer, data.rhs);
+ try writer.writeAll(")");
+ },
+ .@"break" => {
+ const data: Teg.Inst.Break = teg.instData(.@"break", inst).extra_index.get(teg);
+ try writer.writeAll("(");
+ try s.dumpRefMarker(writer, data.block.toRef());
+ try writer.writeAll(", ");
+ try s.dumpRefMarker(writer, data.operand);
+ try writer.writeAll(")");
+ },
+ .string => {
+ const data: Teg.SlicedString = teg.instData(.string, inst).extra_index.get(teg);
+ try tty.setColor(writer, .green);
+ try writer.writeAll(" \"");
+ try writer.writeAll(data.slice(teg.string_bytes));
+ try writer.writeAll("\"");
+ try tty.setColor(writer, .reset);
+ },
+ inline .memory,
+ .discard,
+ .ensure_void,
+ .ensure_bool,
+ .negate,
+ .bool_not,
+ .bit_not,
+ .js,
+ .import,
+ .fetch,
+ .compile_error,
+ => |comptime_tag| {
+ const data = teg.instData(comptime_tag, inst);
+ try writer.writeAll("(");
+ try s.dumpRefMarker(writer, data.operand);
+ try writer.writeAll(")");
+ },
+ .call => {
+ const call: Teg.Inst.Call = teg.instData(.call, inst).extra_index.get(teg);
+ try writer.writeAll("(");
+ try s.dumpRefMarker(writer, call.base);
+ try writer.writeAll(", ");
+ try s.dumpRefMarkerList(writer, call.args.slice(teg), '[', ']');
+ try writer.writeAll(")");
+ },
+ .load_decl => {
+ const call: Teg.Inst.StrNode = teg.instData(.load_decl, inst);
+
+ try tty.setColor(writer, .red);
+ try writer.print(" {s}", .{call.string.slice(teg.string_bytes)});
+ try tty.setColor(writer, .reset);
+ },
+ .@"if" => {
+ const data: Teg.Inst.If = teg.instPayload(.@"if", inst);
+ try writer.writeAll("(");
+ try s.dumpRefMarker(writer, data.cond);
+ try writer.writeAll(") {");
+ s.indent += 1;
+ try s.newline(writer);
+ try writer.writeAll("yes: ");
+ try s.dumpRef(writer, data.yes);
+ try s.newline(writer);
+ try writer.writeAll("no: ");
+ try s.dumpRef(writer, data.no);
+ s.indent -= 1;
+ try s.newline(writer);
+ try writer.writeAll("}");
+ },
+ else => std.debug.panic("TODO format {s}", .{@tagName(tag)}),
+ }
+ }
+
+ fn dumpRef(s: *State, writer: anytype, ref: Teg.Inst.Ref) !void {
+ if (ref.toIndex()) |index| {
+ try s.dumpInst(writer, index);
+ } else {
+ try s.tty.setColor(writer, .blue);
+ try writer.print("%{s}", .{@tagName(ref)});
+ try s.tty.setColor(writer, .reset);
+ }
+ }
+
+ fn dumpRefMarker(s: *State, writer: anytype, ref: Teg.Inst.Ref) !void {
+ if (ref.toIndex()) |index| {
+ try s.tty.setColor(writer, .blue);
+ try writer.print("%{d}", .{@intFromEnum(index)});
+ try s.tty.setColor(writer, .reset);
+ } else {
+ try s.tty.setColor(writer, .blue);
+ try writer.print("%{s}", .{@tagName(ref)});
+ try s.tty.setColor(writer, .reset);
+ }
+ }
+
+ fn dumpInstList(state: *State, writer: anytype, list: []const Teg.Inst.Index, l: u8, r: u8) !void {
+ try writer.print(" {c}", .{l});
+ if (list.len > 0) {
+ state.indent += 1;
+
+ for (list) |inst| {
+ try state.newline(writer);
+ try state.dumpInst(writer, inst);
+ }
+
+ state.indent -= 1;
+ try state.newline(writer);
+ }
+ try writer.print("{c}", .{r});
+ }
+
+ fn dumpRefList(state: *State, writer: anytype, list: []const Teg.Inst.Ref, l: u8, r: u8) !void {
+ try writer.print(" {c}", .{l});
+ if (list.len > 0) {
+ state.indent += 1;
+
+ for (list) |ref| {
+ try state.newline(writer);
+ try state.dumpRef(writer, ref);
+ }
+
+ state.indent -= 1;
+
+ try state.newline(writer);
+ }
+ try writer.print("{c}", .{r});
+ }
+
+ fn dumpRefMarkerList(state: *State, writer: anytype, list: []const Teg.Inst.Ref, l: u8, r: u8) !void {
+ try writer.print(" {c}", .{l});
+ if (list.len > 0) {
+ state.indent += 1;
+
+ for (list) |ref| {
+ try state.newline(writer);
+ try state.dumpRefMarker(writer, ref);
+ }
+
+ state.indent -= 1;
+
+ try state.newline(writer);
+ }
+ try writer.print("{c}", .{r});
+ }
+};
+
+pub fn format(
+ dump: @This(),
+ comptime unused_fmt: []const u8,
+ _: std.fmt.FormatOptions,
+ writer: anytype,
+) !void {
+ comptime assert(unused_fmt.len == 0);
+
+ var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+ defer arena.deinit();
+
+ const teg = dump.teg;
+ var state = State{
+ .teg = &teg,
+ .tty = detectTTYFromWriter(writer),
+ .arena = arena.allocator(),
+ };
+ try state.dumpInst(writer, .main_module_inst);
+ try writer.writeAll("\n");
+ assert(state.indent == 0);
+}
+
+const std = @import("std");
+const assert = std.debug.assert;
+
+const Teg = @import("Teg.zig");
+
+const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter;
diff --git a/src/compiler/Teg.zig b/src/compiler/Teg.zig
new file mode 100644
index 0000000000000000000000000000000000000000..3f38ec22e6b4c167a68c892d39cfd9cadd16e1bd
--- /dev/null
+++ b/src/compiler/Teg.zig
@@ -0,0 +1,435 @@
+//! TODO: rename this to something else. Teg is not a graph
+
+insts: Inst.List.Slice,
+string_bytes: []const u8,
+extra: ExtraStore = .{},
+
+pub const ExtraStore = dod.ExtraStore(.teg, .@"const");
+pub const EncodedData = dod.EncodedData(Inst.Tag, u64);
+pub const ExtraIndex = ExtraStore.Index;
+pub const Dump = @import("Teg.Dump.zig");
+
+pub const Inst = struct {
+ pub const List = std.MultiArrayList(Inst);
+
+ tag: Tag,
+ data: EncodedData,
+
+ // todo: fully document every thing
+ pub const Tag = enum {
+ module,
+ declaration,
+
+ block,
+ /// Creates a memory cell in the current scope.
+ memory,
+ /// Performs a type coersion, such as in variable declarations.
+ /// Emits a sema error if the type conversion fails.
+ /// `let x: i32 = 1`
+ as,
+ load_decl,
+ /// Load a parameter
+ param,
+ store,
+ discard,
+ ensure_void,
+ ensure_bool,
+ /// lowered from `break` or `return`
+ /// ends a block with a resulting value
+ @"break",
+ number,
+ string,
+
+ @"if",
+
+ add,
+ sub,
+ mul,
+ div,
+ rem,
+ pow,
+ @"and",
+ @"or",
+ bit_and,
+ bit_or,
+ bit_xor,
+ shl,
+ shr,
+ eq,
+ ne,
+ lt,
+ le,
+ gt,
+ ge,
+
+ negate,
+ bool_not,
+ bit_not,
+ field,
+ call,
+
+ /// Implements the `@js` builtin
+ js,
+ /// Implements the `@import` builtin
+ import,
+ /// Implements the `@fetch` builtin
+ fetch,
+ /// Implements the `@compileError` builtin
+ compile_error,
+
+ pub fn Data(comptime tag: Tag) type {
+ return switch (tag) {
+ .module => PlNode(IndexList),
+ .declaration => PlNode(Decl),
+ .block => PlNode(IndexList),
+ .as => PlNode(As),
+ .@"break" => PlNode(Break),
+ .number => PlNode(Float),
+ .string => PlNode(SlicedString),
+ .call => PlNode(Call),
+ .@"if" => PlNode(If),
+ .load_decl => StrNode,
+ .param => DataNode,
+
+ // Single instruction operand
+ .memory,
+ .discard,
+ .ensure_void,
+ .ensure_bool,
+ .negate,
+ .bool_not,
+ .bit_not,
+ .js,
+ .import,
+ .fetch,
+ .compile_error,
+ => RefNode,
+
+ // Two instruction operands
+ .field,
+ .store,
+ .add,
+ .sub,
+ .mul,
+ .div,
+ .rem,
+ .pow,
+ .@"and",
+ .@"or",
+ .bit_and,
+ .bit_or,
+ .bit_xor,
+ .shl,
+ .shr,
+ .eq,
+ .ne,
+ .lt,
+ .le,
+ .gt,
+ .ge,
+ => PlNode(Bin),
+ };
+ }
+
+ pub fn airTag(comptime tag: Tag) Air.Inst.Tag {
+ return @field(Air.Inst.Tag, @tagName(tag));
+ }
+ };
+
+ /// TODO: make enum? idk
+ const NodeOffset = i32;
+
+ pub fn PlNode(comptime Payload: type) type {
+ return struct {
+ pub const T = Payload;
+
+ src_node: NodeOffset,
+ extra_index: ExtraIndex(Payload),
+ };
+ }
+
+ pub const RefNode = struct {
+ src_node: NodeOffset,
+ operand: Inst.Ref,
+ };
+
+ pub const StrNode = struct {
+ src_node: NodeOffset,
+ string: NullTerminatedString,
+ };
+
+ pub const DataNode = struct {
+ src_node: NodeOffset,
+ data: u32,
+ };
+
+ /// The position of a Teg instruction within the `Teg` instructions array.
+ pub const Index = enum(u32) {
+ /// Teg is structured so that the outermost "main" struct of any file
+ /// is always at index 0.
+ main_module_inst = 0,
+ _,
+
+ pub const static_len = @intFromEnum(Ref.empty_string) + 1;
+
+ pub fn toRef(i: Index) Inst.Ref {
+ return @enumFromInt(static_len + @intFromEnum(i));
+ }
+
+ pub fn toOptional(i: Index) OptionalIndex {
+ return @enumFromInt(@intFromEnum(i));
+ }
+ };
+
+ pub const OptionalIndex = enum(u32) {
+ /// Teg is structured so that the outermost "main" module of any file
+ /// is always at index 0.
+ main_module_inst = 0,
+ ref_start_index = Index.static_len,
+ none = std.math.maxInt(u32),
+ _,
+
+ pub fn unwrap(oi: OptionalIndex) ?Index {
+ return if (oi == .none) null else @enumFromInt(@intFromEnum(oi));
+ }
+ };
+
+ /// A reference to instruction, or to an InternPool index, or neither.
+ ///
+ /// If the integer tag value is < InternPool.static_len, then it
+ /// corresponds to an InternPool index. Otherwise, this refers to a ZIR
+ /// instruction.
+ ///
+ /// The tag type is specified so that it is safe to bitcast between `[]u32`
+ /// and `[]Ref`.
+ pub const Ref = enum(u32) {
+ opaque_type,
+ bool_type,
+ int32_type,
+ float_type,
+ string_type,
+ void_type,
+ noreturn_type,
+ type_type,
+
+ void_value,
+ bool_true,
+ bool_false,
+ int32_zero,
+ float_zero,
+ nan,
+ empty_string,
+
+ /// This Ref does not correspond to any instruction or constant
+ /// value and may instead be used as a sentinel to indicate null.
+ none = std.math.maxInt(u32),
+
+ _,
+
+ pub fn toIndex(inst: Ref) ?Index {
+ assert(inst != .none);
+ const ref_int = @intFromEnum(inst);
+ if (ref_int >= Index.static_len) {
+ return @enumFromInt(ref_int - Index.static_len);
+ } else {
+ return null;
+ }
+ }
+
+ pub fn toIndexAllowNone(inst: Ref) ?Index {
+ if (inst == .none) return null;
+ return toIndex(inst);
+ }
+
+ pub fn toInterned(inst: Ref) ?InternPool.Index {
+ const ref_int = @intFromEnum(inst);
+
+ // todo: is this > or >=
+ if (ref_int > Index.static_len)
+ return null;
+
+ return @enumFromInt(ref_int);
+ }
+ };
+
+ pub const Bin = struct {
+ lhs: Inst.Ref,
+ rhs: Inst.Ref,
+ };
+
+ pub const As = struct {
+ type: Inst.Ref,
+ operand: Inst.Ref,
+ };
+
+ pub const Break = struct {
+ block: Inst.Index,
+ operand: Inst.Ref,
+ };
+
+ pub const Call = struct {
+ base: Inst.Ref,
+ args: RefList,
+ };
+
+ pub const If = struct {
+ cond: Inst.Ref,
+ yes: Inst.Ref,
+ no: Inst.Ref,
+ };
+
+ /// This will contain trailing data depending on the .kind
+ pub const Decl = struct {
+ name: NullTerminatedString,
+ hash: Cache.BinDigest,
+ kind: Kind,
+ instruction_count: u32,
+
+ pub const Kind = enum {
+ function,
+ // future: struct, let, var, etc
+ };
+ };
+
+ /// Trailng data for function declarations
+ pub const DeclFn = struct {
+ params: RefList,
+ return_type: Ref,
+ body: Ref,
+ };
+};
+
+pub const NullTerminatedString = enum(u32) {
+ empty = 0,
+ _,
+
+ pub fn slice(str: NullTerminatedString, string_bytes: []const u8) [:0]const u8 {
+ if (str == .empty) return "";
+ const sub = string_bytes[@intFromEnum(str)..];
+ const i = mem.indexOfScalar(u8, sub, 0) orelse unreachable;
+
+ return sub[0..i :0];
+ }
+};
+
+pub const SlicedString = struct {
+ start: u32,
+ len: u32,
+
+ pub fn slice(str: SlicedString, string_bytes: []const u8) []const u8 {
+ return string_bytes[str.start..][0..str.len];
+ }
+};
+
+pub const RefList = struct {
+ start: u32,
+ len: u32,
+
+ pub fn slice(self: RefList, teg: *const Teg) []const Inst.Ref {
+ return @ptrCast(teg.extra.list.items[self.start..][0..self.len]);
+ }
+};
+
+pub const IndexList = struct {
+ start: u32,
+ len: u32,
+
+ pub fn slice(self: IndexList, teg: *const Teg) []const Inst.Index {
+ return @ptrCast(teg.extra.list.items[self.start..][0..self.len]);
+ }
+};
+
+pub fn deinit(teg: *Teg, gpa: std.mem.Allocator) void {
+ gpa.free(teg.string_bytes);
+ teg.extra.deinit(gpa);
+ teg.insts.deinit(gpa);
+ teg.* = undefined;
+}
+
+pub fn instTag(teg: *const Teg, index: Inst.Index) Inst.Tag {
+ return teg.insts.items(.tag)[@intFromEnum(index)];
+}
+
+pub fn instEncodedData(teg: *const Teg, index: Inst.Index) EncodedData {
+ return teg.insts.items(.data)[@intFromEnum(index)];
+}
+
+pub fn instData(teg: *const Teg, comptime tag: Inst.Tag, index: Inst.Index) tag.Data() {
+ return teg.instEncodedData(index).decode(tag, teg.instTag(index));
+}
+
+pub fn instPayload(teg: *const Teg, comptime tag: Inst.Tag, index: Inst.Index) tag.Data().T {
+ return teg.instData(tag, index).extra_index.get(teg);
+}
+
+pub fn blockInstructions(teg: *const Teg, index: Inst.Index) []const Inst.Index {
+ return teg.instData(.block, index).extra_index.get(teg).slice(teg);
+}
+
+pub fn fullFuncDecl(teg: *const Teg, index: Inst.Index) struct { Inst.Decl, Inst.DeclFn } {
+ const decl, var trail = teg.instData(.declaration, index).extra_index.getTrail(teg);
+ const decl_fn = trail.read(&teg.extra, Inst.DeclFn);
+ return .{ decl, decl_fn };
+}
+
+pub fn declData(teg: *const Teg, index: Inst.Index) Inst.Decl {
+ return teg.instData(.declaration, index).extra_index.get(teg);
+}
+
+pub fn moduleDeclarations(teg: *const Teg, module_index: Inst.Index) []const Inst.Index {
+ const module = teg.instData(.module, module_index);
+ const module_data: IndexList = module.extra_index.get(teg);
+ const slice = module_data.slice(teg);
+ // if (std.debug.runtime_safety) {
+ // for (slice) |item| {
+ // if (teg.instTag(item) != .declaration) {
+ // std.debug.print("\ngot unknown tag {s}\n", .{@tagName(teg.instTag(item))});
+ // unreachable;
+ // }
+ // }
+ // }
+ return slice;
+}
+
+pub fn rootFunction(teg: *const Teg) Inst.Index {
+ const decls = teg.moduleDeclarations(.main_module_inst);
+ const root_fn = decls[decls.len - 1];
+ assert(teg.declName(root_fn).len == 0);
+ return root_fn;
+}
+
+pub fn declName(teg: *const Teg, decl_index: Inst.Index) []const u8 {
+ const decl = teg.instData(.declaration, decl_index);
+ const decl_data: Inst.Decl = decl.extra_index.get(teg);
+ return decl_data.name.slice(teg.string_bytes);
+}
+
+pub fn dumpToWriter(teg: Teg, writer: anytype) !void {
+ try writer.print("{}", .{Dump{ .teg = teg }});
+}
+
+const Teg = @This();
+
+const std = @import("std");
+const mem = std.mem;
+const assert = std.debug.assert;
+
+const Cache = @import("Cache.zig");
+
+const Ast = @import("Ast.zig");
+const Unused = Ast.Unused;
+const NodeIndex = Ast.NodeIndex;
+const TokenIndex = Ast.TokenIndex;
+const OptionalNodeIndex = Ast.OptionalNodeIndex;
+const OptionalTokenIndex = Ast.OptionalTokenIndex;
+const OptionalExtraIndex = Ast.OptionalExtraIndex;
+const optionalTokenIndex = OptionalTokenIndex.init;
+const optionalNodeIndex = OptionalNodeIndex.init;
+const optionalExtraIndex = OptionalExtraIndex.init;
+
+const dod = @import("dod.zig");
+const Float = dod.Float;
+
+const InternPool = @import("InternPool.zig");
+
+const Air = @import("Air.zig");
diff --git a/src/compiler/Tokenizer.zig b/src/compiler/Tokenizer.zig
new file mode 100644
index 0000000000000000000000000000000000000000..546c06e330ed999ccd554849c9515479905f0214
--- /dev/null
+++ b/src/compiler/Tokenizer.zig
@@ -0,0 +1,913 @@
+//! The tokenizer is extremely simple structure that, given a view of source
+//! code, can stream tokens one at a time. A token is an 8-bit tag and a 32-bit
+//! offset to where that token is located.
+//!
+//! Retriving payloads like string/numeric values, identifier names, etc is
+//! done separately, either in Parse or Transform, when the respective data
+//! is actually needed. Tokenization does guarante that such data is valid.
+
+// TODO: make `code` a [:0]const u8. this doesnt disallow null bytes, but
+// it requires the source to end with one. This can make byteAt(1) return u8
+// instead of an optional. I am told this improves performance.
+code: []const u8,
+pos: ByteOffset = 0,
+
+/// Tokenize source code into a list of tokens.
+/// When an error occurs, the position of the error is returned in `error_pos`.
+/// Caller owns returned slice.
+pub fn tokenize(
+ source_code: []const u8,
+ gpa: mem.Allocator,
+ error_pos: *ByteOffset,
+) TokenizeError!Token.List.Slice {
+ const trace = T.trace(@src());
+ defer trace.end();
+
+ var tokens = Token.List{};
+ errdefer tokens.deinit(gpa);
+
+ // pre-allocate some space based on a very rough guess
+ try tokens.ensureTotalCapacity(gpa, @divFloor(source_code.len, 8));
+
+ var tokenizer: Tokenizer = .{ .code = source_code, .pos = 0 };
+ errdefer error_pos.* = tokenizer.pos;
+
+ tokenizer.skipWhitespace();
+ while (try tokenizer.next()) |token| {
+ try tokens.append(gpa, token);
+ tokenizer.skipWhitespace();
+ }
+
+ try tokens.append(gpa, .{
+ .tag = .eof,
+ .offset = @intCast(source_code.len),
+ });
+
+ return tokens.toOwnedSlice();
+}
+
+pub const ByteOffset = u32;
+
+pub const Token = struct {
+ tag: Tag,
+ offset: ByteOffset,
+
+ pub const List = std.MultiArrayList(Token);
+
+ pub const Tag = enum {
+ id,
+ builtin_id,
+ number_int,
+ number_float,
+ string,
+ multiline_string,
+ doc_comment,
+ module_doc_comment,
+
+ // sort-lines: start
+ kw_assert,
+ kw_break,
+ kw_comptime,
+ kw_defer,
+ kw_else,
+ kw_enum,
+ kw_fn,
+ kw_infer,
+ kw_for,
+ kw_while,
+ kw_if,
+ kw_impl,
+ kw_json,
+ kw_let,
+ kw_match,
+ kw_module,
+ kw_pub,
+ kw_pure,
+ kw_return,
+ kw_struct,
+ kw_trait,
+ kw_var,
+ // sort-lines: end
+
+ // sort-lines: start
+ @"and",
+ and_and,
+ and_and_equal,
+ and_equal,
+ bang,
+ bang_equal,
+ caret,
+ caret_equal,
+ colon,
+ comma,
+ dot,
+ end_brace,
+ end_bracket,
+ end_paren,
+ equal,
+ equal_equal,
+ equal_lt,
+ gt,
+ gt_equal,
+ gt_gt,
+ gt_gt_equal,
+ gt_gt_gt,
+ gt_gt_gt_equal,
+ lt,
+ lt_equal,
+ lt_lt,
+ lt_lt_equal,
+ minus,
+ minus_equal,
+ minus_lt,
+ percent,
+ percent_equal,
+ pipe,
+ pipe_equal,
+ pipe_pipe,
+ pipe_pipe_equal,
+ plus,
+ plus_equal,
+ semi,
+ slash,
+ slash_equal,
+ star,
+ star_equal,
+ star_star,
+ star_star_equal,
+ start_brace,
+ start_bracket,
+ start_paren,
+ tilde,
+ tilde_equal,
+ // sort-lines: end
+
+ /// Represents a tokenization error. Instead of reporting an error during
+ /// tokenization, parsing will begin, and an error will be reported when
+ /// this token is hit. That way, the error can tell you what it expected.
+ invalid,
+ /// End of file
+ /// Used so that parser does not need to do out of bounds checks
+ eof,
+
+ /// Most tokens have a static length, such as "struct" or "+="
+ pub fn staticLen(tag: Tag) ?u32 {
+ return switch (tag) {
+ // do not have a static length
+ .id,
+ .builtin_id,
+ .number_int,
+ .number_float,
+ .string,
+ .multiline_string,
+ .doc_comment,
+ .module_doc_comment,
+ => null,
+
+ .@"and" => 1,
+ .and_and => 2,
+ .bang => 1,
+ .bang_equal => 2,
+ .equal_lt => 2,
+ .caret => 1,
+ .end_brace => 1,
+ .end_bracket => 1,
+ .end_paren => 1,
+ .colon => 1,
+ .comma => 1,
+ .dot => 1,
+ .equal => 1,
+ .equal_equal => 2,
+ .gt => 1,
+ .gt_equal => 2,
+ .gt_gt => 2,
+ .gt_gt_gt => 3,
+ .lt => 1,
+ .lt_equal => 2,
+ .lt_lt => 2,
+ .minus => 1,
+ .start_brace => 1,
+ .start_bracket => 1,
+ .start_paren => 1,
+ .pipe => 1,
+ .pipe_pipe => 2,
+ .percent => 1,
+ .plus => 1,
+ .minus_lt => 2,
+ .semi => 1,
+ .slash => 1,
+ .star => 1,
+ .star_star => 2,
+ .tilde => 1,
+ .slash_equal => 2,
+ .percent_equal => 2,
+ .tilde_equal => 2,
+ .caret_equal => 2,
+ .and_equal => 2,
+ .pipe_equal => 2,
+ .lt_lt_equal => 3,
+ .gt_gt_equal => 3,
+ .gt_gt_gt_equal => 4,
+ .plus_equal => 2,
+ .minus_equal => 2,
+ .star_equal => 2,
+ .star_star_equal => 3,
+ .pipe_pipe_equal => 3,
+ .and_and_equal => 3,
+
+ .invalid => 0,
+ .eof => 0,
+
+ inline else => |keyword| comptime len: {
+ if (!mem.startsWith(u8, @tagName(keyword), "kw_")) {
+ @compileError("Token tag is not a keyword: " ++ @tagName(keyword));
+ }
+ break :len @tagName(keyword).len - 3;
+ },
+ };
+ }
+ };
+
+ /// Instead of storing the length of a token, we can recompute this
+ /// information on demand, as it is not often needed.
+ ///
+ /// If you are computing the length of something that is knowable
+ /// without the source code, use token.tag.staticLen() instead.
+ pub fn len(token: Token, utf8_source: []const u8) u32 {
+ return token.tag.staticLen() orelse {
+ var tokenizer = Tokenizer{ .code = utf8_source, .pos = token.offset };
+ assert(
+ (tokenizer.next() catch
+ unreachable // data corruption
+ orelse unreachable // data corruption
+ ).tag == token.tag,
+ );
+ return tokenizer.pos - token.offset;
+ };
+ }
+
+ pub fn slice(token: Token, utf8_source: []const u8) []const u8 {
+ return utf8_source[token.offset..][0..token.len(utf8_source)];
+ }
+
+ const keywords = std.ComptimeStringMap(Tag, keywords: {
+ const fields = std.enums.values(Tag);
+ const KV = struct { []const u8, Tag };
+ var kws: [fields.len]KV = undefined;
+ var i = 0;
+ for (fields) |field| {
+ if (mem.startsWith(u8, @tagName(field), "kw_")) {
+ kws[i] = .{ @tagName(field)[3..], field };
+ i += 1;
+ }
+ }
+ break :keywords kws[0..i];
+ });
+};
+
+/// When any of these error occurs, the tokenizer's position is set to
+/// the start of the offending token.
+pub const NextTokenError = error{
+ /// Generic token syntax error
+ InvalidSyntax,
+ /// A trailing '_' was seen in a numeric literal.
+ TrailingNumericSeparator,
+ /// Newline or EOF encountered while parsing a string literal.
+ UnterminatedString,
+ /// A string escape sequence was invalid.
+ InvalidEscapeSequence,
+ /// The codepoint represented by a unicode escape sequence
+ /// is not a valid unicode codepoint.
+ InvalidEscapeCodepoint,
+ /// There are unresolved git conflicts in the source code.
+ GitConflict,
+ /// The source code is not valid UTF-8.
+ InvalidUtf8,
+};
+
+pub const TokenizeError = mem.Allocator.Error || NextTokenError;
+
+/// Next assumes the position is already at the start of a token.
+/// You must call `skipWhitespace` before calling this.
+pub fn next(t: *Tokenizer) !?Token {
+ while (true) {
+ const trace = T.trace(@src());
+ defer trace.end();
+
+ switch (t.byte() orelse return null) {
+ ' ', '\t', '\n', '\r' => unreachable, // call skipWhitespace
+
+ 'a'...'z', 'A'...'Z', '_', '$' => {
+ const start = t.pos;
+ t.pos += 1;
+ return try t.tokenizeIdentifier(start);
+ },
+
+ '@' => switch (t.byteAt(1) orelse 0) {
+ '"' => {
+ const start = t.pos;
+ t.pos += 1;
+ try t.consumeString();
+ return .{ .tag = .id, .offset = start };
+ },
+ else => {
+ const start = t.pos;
+ t.pos += 1;
+ return try t.tokenizeIdentifier(start);
+ },
+ },
+
+ '0'...'9' => return try t.tokenizeNumber(),
+
+ '.' => switch (t.byteAt(1) orelse 0) {
+ '0'...'9' => return try t.tokenizeNumber(),
+ else => return t.simpleToken(.dot),
+ },
+
+ '/' => switch (t.byteAt(1) orelse 0) {
+ '=' => return t.simpleToken(.slash_equal),
+ '/' => {
+ if ((t.byteAt(2) orelse 0 == '/') and
+ (t.byteAt(3) orelse 0 != '/'))
+ {
+ const start = t.pos;
+ t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 3, '\n') orelse t.code.len);
+ return .{ .tag = .doc_comment, .offset = start };
+ }
+
+ if (t.byteAt(2) orelse 0 == '!') {
+ const start = t.pos;
+ t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len);
+ return .{ .tag = .module_doc_comment, .offset = start };
+ }
+
+ // skip the comment
+ t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len);
+ t.skipWhitespace();
+ continue;
+ },
+ else => return t.simpleToken(.slash),
+ },
+
+ '\\' => switch (t.byteAt(1) orelse 0) {
+ '\\' => {
+ const start = t.pos;
+ t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len);
+ return .{ .tag = .multiline_string, .offset = start };
+ },
+ else => return t.maybeUTF8Error(error.InvalidSyntax),
+ },
+
+ '"' => return try t.tokenizeString(),
+
+ '&' => switch (t.byteAt(1) orelse 0) {
+ '&' => return t.simpleToken(.and_and),
+ '=' => return t.simpleToken(.and_equal),
+ else => return t.simpleToken(.@"and"),
+ },
+ '|' => switch (t.byteAt(1) orelse 0) {
+ '|' => return t.simpleToken(.pipe_pipe),
+ '=' => return t.simpleToken(.pipe_equal),
+ else => return t.simpleToken(.pipe),
+ },
+ '^' => switch (t.byteAt(1) orelse 0) {
+ '^' => return t.simpleToken(.caret),
+ '=' => return t.simpleToken(.caret_equal),
+ else => return t.simpleToken(.caret),
+ },
+ '!' => switch (t.byteAt(1) orelse 0) {
+ '=' => return t.simpleToken(.equal_equal),
+ else => return t.simpleToken(.bang_equal),
+ },
+ '=' => switch (t.byteAt(1) orelse 0) {
+ '=' => return t.simpleToken(.equal_equal),
+ else => return t.simpleToken(.equal),
+ },
+ '>' => switch (t.byteAt(1) orelse 0) {
+ '=' => return t.simpleToken(.gt_equal),
+ '>' => switch (t.byteAt(2) orelse 0) {
+ '>' => return t.simpleToken(.gt_gt_gt),
+ else => return t.simpleToken(.gt_gt),
+ },
+ else => return t.simpleToken(.gt),
+ },
+ '<' => switch (t.byteAt(1) orelse 0) {
+ '=' => return t.simpleToken(.lt_equal),
+ '<' => switch (t.byteAt(2) orelse 0) {
+ '=' => return t.simpleToken(.lt_lt_equal),
+ else => {
+ // Git merge conflicts are <<<<<<<
+ const start = t.pos;
+ t.pos += 2;
+ brk: {
+ var remain: u3 = 4;
+ while (t.byte() orelse break :brk == '<') {
+ t.pos += 1;
+ remain -= 1;
+ if (remain == 0) {
+ return error.GitConflict;
+ }
+ }
+ }
+ t.pos = start;
+ return t.simpleToken(.lt_lt);
+ },
+ },
+ else => return t.simpleToken(.lt),
+ },
+ '+' => switch (t.byteAt(1) orelse 0) {
+ '=' => return t.simpleToken(.plus_equal),
+ else => return t.simpleToken(.plus),
+ },
+ '-' => switch (t.byteAt(1) orelse 0) {
+ '=' => return t.simpleToken(.minus_equal),
+ '>' => return t.simpleToken(.minus_lt),
+ else => return t.simpleToken(.minus),
+ },
+ '*' => switch (t.byteAt(1) orelse 0) {
+ '*' => return t.simpleToken(.star_star),
+ '=' => return t.simpleToken(.star_equal),
+ else => return t.simpleToken(.star),
+ },
+ '%' => switch (t.byteAt(1) orelse 0) {
+ '=' => return t.simpleToken(.percent_equal),
+ else => return t.simpleToken(.percent),
+ },
+ '~' => switch (t.byteAt(1) orelse 0) {
+ '=' => return t.simpleToken(.tilde_equal),
+ else => return t.simpleToken(.tilde),
+ },
+
+ ';' => return t.simpleToken(.semi),
+ ':' => return t.simpleToken(.colon),
+ '(' => return t.simpleToken(.start_paren),
+ ')' => return t.simpleToken(.end_paren),
+ '[' => return t.simpleToken(.start_bracket),
+ ']' => return t.simpleToken(.end_bracket),
+ '{' => return t.simpleToken(.start_brace),
+ '}' => return t.simpleToken(.end_brace),
+ ',' => return t.simpleToken(.comma),
+
+ 0x80...0xFF => {
+ // need to decode utf-8
+ const byte_len, const code_point = try t.codePoint();
+
+ if (unicode_props.isXidStart(code_point)) {
+ const start = t.pos;
+ t.pos += byte_len;
+ return try t.tokenizeIdentifier(start);
+ }
+
+ return error.InvalidSyntax;
+ },
+
+ else => return error.InvalidSyntax,
+ }
+ }
+}
+
+fn byte(t: Tokenizer) ?u8 {
+ if (t.pos >= t.code.len) return null;
+ return t.code[t.pos];
+}
+
+/// Remember that using this only makes sense if you know for certain that all
+/// previous bytes are ASCII, otherwise you may hit UB
+fn byteAt(t: Tokenizer, offset: u32) ?u8 {
+ if (t.pos + offset >= t.code.len) return null;
+ // assert that the previous bytes are ASCII
+ // current byte is allowed to be a continuation byte
+ for (t.code[t.pos .. t.pos + offset]) |b| {
+ assert(b < 0x80);
+ }
+ return t.code[t.pos + offset];
+}
+
+fn codePoint(t: Tokenizer) !struct { u3, u21 } {
+ const b = t.byte() orelse unreachable; // must check the first byte manually
+ assert(b >= 0x80); // must be a continuation byte
+
+ const byte_len = unicode.utf8ByteSequenceLength(b) catch
+ return error.InvalidUtf8;
+ assert(byte_len > 1);
+
+ const code_point = unicode.utf8Decode(t.code[t.pos..][0..byte_len]) catch
+ return error.InvalidUtf8;
+
+ return .{ byte_len, code_point };
+}
+
+fn isAtIdentifierStart(t: *Tokenizer) !bool {
+ switch (t.byte() orelse return false) {
+ 'a'...'z', 'A'...'Z', '_', '$' => return true,
+ else => return false,
+ 0x80...0xFF => {
+ _, const code_point = try t.codePoint();
+ return unicode_props.isXidStart(code_point);
+ },
+ }
+}
+
+fn tokenizeIdentifier(t: *Tokenizer, start_byte: u32) !Token {
+ const trace = T.trace(@src());
+ defer trace.end();
+
+ while (true) {
+ switch (t.byte() orelse break) {
+ 'a'...'z', 'A'...'Z', '0'...'9', '_', '$' => {
+ t.pos += 1;
+ },
+ 0x80...0xFF => {
+ const byte_len, const code_point = try t.codePoint();
+
+ t.pos += byte_len;
+
+ if (!unicode_props.isXidContinue(code_point)) {
+ break;
+ }
+ },
+ else => break,
+ }
+ }
+
+ return .{
+ .tag = Token.keywords.get(t.code[start_byte..t.pos]) orelse
+ if (t.code[start_byte] == '@')
+ .builtin_id
+ else
+ .id,
+ .offset = start_byte,
+ };
+}
+
+/// Numbers have the following rules:
+/// - May start with a `.`, but cannot end with `.`
+/// - May contain leading zeros, but transform pass will warn.
+/// - May contain a `_` separator, cannot have two in a row.
+/// - May contain `e` or `E` for exponents.
+/// - May use 0x, 0b, 0o prefixes for hex, binary, octal numbers.
+/// The token is tagged as `number_int` or `number_float` depending
+/// on if it contained a `.` This is to help later re-parses.
+//
+// TODO: is it a zig compiler bug that `?` is required here?
+fn tokenizeNumber(t: *Tokenizer) !?Token {
+ const trace = T.trace(@src());
+ defer trace.end();
+
+ // Caller ensures the following invariant:
+ switch (t.byte() orelse unreachable) {
+ '.' => switch (t.byteAt(1) orelse unreachable) {
+ '0'...'9' => {},
+ else => unreachable,
+ },
+ '0'...'9' => {},
+ else => unreachable,
+ }
+
+ // Parse Hex, Binary, Octal numbers
+ if (t.byte() == '0') {
+ switch (t.byteAt(1) orelse 0) {
+ 'x', 'X' => {
+ t.pos += 2;
+ try t.consumeIntegerRadix(.hex);
+ },
+ 'b', 'B' => {
+ t.pos += 2;
+ try t.consumeIntegerRadix(.binary);
+ },
+ 'o', 'O' => {
+ t.pos += 2;
+ try t.consumeIntegerRadix(.octal);
+ },
+ else => {},
+ }
+ }
+
+ const start = t.pos;
+ var tag: Token.Tag = .number_int;
+
+ if (t.byte() == '.') {
+ t.pos += 1;
+ tag = .number_float;
+ try t.consumeInteger();
+ } else {
+ try t.consumeInteger();
+
+ // decimal point notation
+ switch (t.byte() orelse 0) {
+ '.' => switch (t.byteAt(1) orelse 0) {
+ '0'...'9' => {
+ t.pos += 1;
+ tag = .number_float;
+ try t.consumeInteger();
+ },
+ else => {
+ // we have not advanced the tokenizer past the `.`
+ assert(tag == .number_int);
+ return .{ .tag = tag, .offset = start };
+ },
+ },
+ else => {},
+ }
+ }
+
+ switch (t.byte() orelse 0) {
+ 'e', 'E' => {
+ tag = .number_float;
+ t.pos += 1;
+
+ switch (t.byte() orelse 0) {
+ '+', '-' => {
+ t.pos += 1;
+ switch (t.byte() orelse 0) {
+ '0'...'9' => {},
+ else => return t.maybeUTF8Error(error.InvalidSyntax),
+ }
+ },
+ '0'...'9' => {},
+ else => return t.maybeUTF8Error(error.InvalidSyntax),
+ }
+
+ try t.consumeInteger();
+
+ // decimal point notation
+ switch (t.byte() orelse 0) {
+ '.' => switch (t.byteAt(1) orelse 0) {
+ '0'...'9' => {
+ t.pos += 1;
+ tag = .number_float;
+ try t.consumeInteger();
+ },
+ else => {},
+ },
+ else => {},
+ }
+ },
+ else => {},
+ }
+
+ // after numbers must be a valid word break the above returns are right
+ // before parsing `.` so those can be unchecked.
+ if (try t.isAtIdentifierStart()) {
+ return error.InvalidSyntax;
+ }
+
+ return .{
+ .tag = tag,
+ .offset = start,
+ };
+}
+
+fn maybeConsumeInteger(t: *Tokenizer) !?void {
+ const trace = T.trace(@src());
+ defer trace.end();
+
+ switch (t.byte() orelse return null) {
+ '0'...'9' => {
+ t.pos += 1;
+ },
+ else => return null,
+ }
+ t.consumeInteger();
+}
+
+/// Requires the first byte to be a digit
+fn consumeInteger(t: *Tokenizer) !void {
+ const trace = T.trace(@src());
+ defer trace.end();
+
+ switch (t.byte() orelse unreachable) {
+ '0'...'9' => {
+ t.pos += 1;
+ },
+ else => unreachable,
+ }
+
+ while (true) {
+ switch (t.byte() orelse return) {
+ '0'...'9' => {
+ t.pos += 1;
+ },
+ '_' => {
+ t.pos += 1;
+ switch (t.byte() orelse 0) {
+ '0'...'9' => {
+ t.pos += 1;
+ },
+ else => return t.maybeUTF8Error(error.TrailingNumericSeparator),
+ }
+ },
+ else => break,
+ }
+ }
+}
+
+const Radix = enum { hex, binary, octal };
+
+fn consumeIntegerRadix(t: *Tokenizer, comptime radix: Radix) !void {
+ const trace = T.trace(@src());
+ defer trace.end();
+
+ const digits = switch (radix) {
+ .hex => "0123456789abcdefABCDEF",
+ .binary => "01",
+ .octal => "01234567",
+ };
+
+ first: {
+ const b = t.byte() orelse return;
+ inline for (digits) |digit| {
+ if (b == digit) {
+ break :first;
+ }
+ }
+ return t.maybeUTF8Error(error.InvalidSyntax);
+ }
+
+ t.pos += 1;
+
+ while (true) {
+ brk: {
+ const b = t.byte() orelse return;
+ inline for (digits) |digit| {
+ if (b == digit) {
+ break :brk;
+ }
+ }
+ return;
+ }
+ t.pos += 1;
+ }
+}
+
+fn tokenizeString(t: *Tokenizer) !Token {
+ assert(t.byte().? == '"');
+ const start = t.pos;
+ try t.consumeString();
+ return .{ .tag = .string, .offset = start };
+}
+
+fn consumeString(t: *Tokenizer) !void {
+ const trace = T.trace(@src());
+ defer trace.end();
+
+ assert(t.byte().? == '"');
+
+ t.pos += 1;
+
+ while (true) {
+ const b = t.byte() orelse return error.UnterminatedString;
+ switch (b) {
+ '"' => {
+ t.pos += 1;
+ break;
+ },
+ '\\' => {
+ switch (t.byteAt(1) orelse {
+ // we want error.InvalidEscapeSequence to be at the '\' char,
+ // but we want error.UnterminatedString to be at the end of the source code
+ t.pos += 1;
+ return error.UnterminatedString;
+ }) {
+ '\n' => return error.UnterminatedString,
+ '\\' => t.pos += 2,
+ '"' => t.pos += 2,
+ 'n' => t.pos += 2,
+ 'r' => t.pos += 2,
+ 't' => t.pos += 2,
+ '0' => switch (t.byteAt(2) orelse {
+ t.pos += 2;
+ return error.UnterminatedString;
+ }) {
+ // '\0xFF' hex escape
+ 'x' => {},
+ // '\0' is special cased
+ else => t.pos += 2,
+ },
+ 'u' => {
+ const esc_start = t.pos;
+ t.consumeStringUnicodeEscape() catch |err| switch (err) {
+ error.InvalidEscapeSequence => {
+ if (t.byte().? == '\n')
+ return error.UnterminatedString;
+ t.pos = esc_start;
+ return err;
+ },
+ error.InvalidEscapeCodepoint => {
+ t.pos = esc_start;
+ return err;
+ },
+ error.UnterminatedString => return err,
+ };
+ },
+ else => return t.maybeUTF8ErrorAt(error.InvalidEscapeSequence, 1),
+ }
+ },
+ else => {
+ t.pos += 1;
+ },
+ 0x80...0xFF => {
+ const byte_len, _ = try t.codePoint();
+ t.pos += byte_len;
+ },
+ }
+ }
+}
+
+fn consumeStringUnicodeEscape(t: *Tokenizer) !void {
+ const trace = T.trace(@src());
+ defer trace.end();
+
+ assert(t.byte().? == '\\');
+ assert(t.byteAt(1).? == 'u');
+ t.pos += 2;
+
+ if ((t.byte() orelse return error.UnterminatedString) != '{')
+ return error.InvalidEscapeSequence;
+ t.pos += 1;
+
+ if ((t.byte() orelse return error.UnterminatedString) == '0') {
+ if ((t.byteAt(1) orelse return error.UnterminatedString) == 'x') {
+ t.pos += 2;
+ }
+ }
+
+ const start = t.pos;
+ t.consumeIntegerRadix(.hex) catch return error.InvalidEscapeSequence;
+
+ const slice = t.code[start..t.pos];
+
+ if ((t.byte() orelse return error.UnterminatedString) != '}')
+ return error.InvalidEscapeSequence;
+
+ const code_point = std.fmt.parseInt(u21, slice, 16) catch |err| switch (err) {
+ error.Overflow => return error.InvalidEscapeCodepoint,
+ error.InvalidCharacter => unreachable,
+ };
+ _ = std.unicode.utf8CodepointSequenceLength(code_point) catch |err| switch (err) {
+ error.CodepointTooLarge => return error.InvalidEscapeCodepoint,
+ };
+
+ t.pos += 1;
+}
+
+fn simpleToken(t: *Tokenizer, comptime tag: Token.Tag) Token {
+ const start = t.pos;
+ t.pos += comptime (tag.staticLen() orelse unreachable);
+ return .{ .tag = tag, .offset = start };
+}
+
+/// We are about to return an error, such as error.TrailingNumericSeparator.
+/// However, the character we ran into might actually be invalid UTF-8.
+/// This utility just checks for that to make sure we dont mask error.InvalidUtf8
+fn maybeUTF8Error(t: *Tokenizer, comptime real_err: anytype) @Type(.{
+ .ErrorSet = &[_]std.builtin.Type.Error{
+ .{ .name = "InvalidUtf8" },
+ .{ .name = @errorName(real_err) },
+ },
+}) {
+ switch (t.byte() orelse return real_err) {
+ 0x80...0xFF => {
+ _ = try t.codePoint();
+ return real_err;
+ },
+ else => return real_err,
+ }
+}
+
+fn maybeUTF8ErrorAt(t: *Tokenizer, comptime real_err: anytype, at: u32) @Type(.{
+ .ErrorSet = &[_]std.builtin.Type.Error{
+ .{ .name = "InvalidUtf8" },
+ .{ .name = @errorName(real_err) },
+ },
+}) {
+ switch (t.byteAt(at) orelse return real_err) {
+ 0x80...0xFF => {
+ _ = try t.codePoint();
+ return real_err;
+ },
+ else => return real_err,
+ }
+}
+
+pub fn skipWhitespace(t: *Tokenizer) void {
+ const trace = T.trace(@src());
+ defer trace.end();
+
+ while (true) {
+ switch (t.byte() orelse break) {
+ ' ', '\t', '\n', '\r' => {
+ t.pos += 1;
+ },
+ else => break,
+ }
+ }
+}
+
+const Tokenizer = @This();
+const std = @import("std");
+const unicode = std.unicode;
+const assert = std.debug.assert;
+const mem = std.mem;
+const unicode_props = @import("unicode_props.zig");
+const T = @import("tracer");
diff --git a/src/compiler/Transform.zig b/src/compiler/Transform.zig
new file mode 100644
index 0000000000000000000000000000000000000000..d2534f57bfed4f540e9d98878f8feb1220ca3932
--- /dev/null
+++ b/src/compiler/Transform.zig
@@ -0,0 +1,1276 @@
+//! Transform pass
+gpa: Allocator,
+ast: Ast,
+insts: Inst.List = .{},
+string_bytes: std.ArrayListUnmanaged(u8) = .{},
+extra: ExtraStore = .{},
+
+scratch: std.ArrayListUnmanaged(u32) = .{},
+
+// TODO: refactor this away
+// The new home for this memory should be pre-allocated on scratch,
+// but then remove the elements, moving scratch items on top of this
+// backwards a few.
+call_args: std.ArrayListUnmanaged(Inst.Ref) = .{},
+scopes: std.ArrayListUnmanaged(Scope) = .{},
+
+pub fn transform(ast: Ast, gpa: Allocator) !Teg {
+ const trace = tracer.trace(@src());
+ defer trace.end();
+
+ var t: Transform = .{
+ .ast = ast,
+ .gpa = gpa,
+ };
+ errdefer t.deinit();
+
+ try t.string_bytes.append(t.gpa, 0);
+
+ const module = try t.transformModule(0);
+ assert(@intFromEnum(module) == 0);
+
+ try t.insts.setCapacity(t.gpa, t.insts.len);
+
+ defer t.deinit();
+
+ return Teg{
+ .insts = t.insts.toOwnedSlice(),
+ .string_bytes = try t.string_bytes.toOwnedSlice(t.gpa),
+ .extra = try t.extra.toOwnedStore(t.gpa),
+ };
+}
+
+pub fn deinit(t: *Transform) void {
+ t.scratch.deinit(t.gpa);
+ t.call_args.deinit(t.gpa);
+ assert(t.scopes.items.len == 0);
+ t.scopes.deinit(t.gpa);
+ t.* = undefined;
+}
+
+const Scope = struct {
+ symbols: std.StringArrayHashMapUnmanaged(Symbol) = .{},
+
+ const Symbol = struct {
+ node: NodeIndex,
+ data: Symbol.Data,
+ used: bool = false,
+ mutated: bool = false,
+
+ const Data = union(enum) {
+ declaration,
+ param,
+ mutable: Inst.Ref,
+ immutable: Inst.Ref,
+ };
+ };
+
+ fn deinit(s: *Scope, gpa: Allocator) void {
+ s.symbols.deinit(gpa);
+ s.* = undefined;
+ }
+};
+
+const ScratchType = enum {
+ ref,
+ index,
+};
+
+const BlockContext = struct {
+ node_base: i32,
+ return_target: ?*?Inst.Index = null,
+ break_target: ?*BreakTarget = null,
+
+ /// This struct is always stored on the stack; the idea is to pass
+ /// mutable pointers to it. When `break :brk` is encountered
+ const BreakTarget = struct {
+ label: ?[]const u8,
+ /// This allows `continue` as well as implicit break targetting.
+ is_loop: bool,
+ /// If `null`, this label was unused.
+ block_index: ?Inst.Index = null,
+ parent: ?*BreakTarget = null,
+
+ fn addChild(parent: *BreakTarget, label: ?[]const u8, is_loop: bool) BreakTarget {
+ return .{
+ .parent = parent,
+ .label = label,
+ .is_loop = is_loop,
+ };
+ }
+
+ fn getInst(target: *BreakTarget, t: *Transform) !Inst.Index {
+ return target.block_index orelse {
+ const inst = try t.addRootInst(.{
+ .tag = .block,
+ .data = undefined,
+ });
+ target.block_index = inst;
+ return inst;
+ };
+ }
+ };
+
+ fn init(root_node: NodeIndex) BlockContext {
+ return .{
+ .node_base = @intCast(root_node),
+ };
+ }
+
+ fn nodeOffset(self: BlockContext, node: NodeIndex) i32 {
+ return @as(i32, @intCast(node)) - self.node_base;
+ }
+
+ fn declOffset(self: BlockContext, node: NodeIndex) BlockContext {
+ return .{
+ .node_base = self.nodeOffset(node),
+ };
+ }
+
+ fn withReturn(self: BlockContext, return_target: ?*?Inst.Index) BlockContext {
+ return .{
+ .node_base = self.node_base,
+ .return_target = return_target,
+ };
+ }
+
+ fn withBreak(self: BlockContext, break_target: *BreakTarget) BlockContext {
+ return .{
+ .node_base = self.node_base,
+ .return_target = self.return_target,
+ .break_target = break_target,
+ };
+ }
+
+ fn findAndMarkBreak(self: BlockContext, t: *Transform, label: []const u8) !?Inst.Index {
+ var break_target = self.break_target;
+ while (break_target) |target| : (break_target = target.parent) {
+ if (target.label) |target_label| {
+ if (mem.eql(u8, label, target_label)) {
+ return try target.getInst(t);
+ }
+ }
+ }
+ return null;
+ }
+};
+
+const Liveness = enum { alive, dead };
+
+const BuiltInIds = std.ComptimeStringMap(Inst.Ref, .{
+ .{ "void", .void_value },
+ .{ "true", .bool_true },
+ .{ "false", .bool_false },
+ .{ "int", .int32_type },
+ .{ "float", .float_type },
+ .{ "string", .string_type },
+ // .{ "bigint", .bigint_type },
+ .{ "bool", .bool_type },
+ .{ "noreturn", .noreturn_type },
+ .{ "type", .type_type },
+ // todo: nan/infinity?
+});
+
+fn transformModule(t: *Transform, module_node: NodeIndex) !Inst.Index {
+ const trace = tracer.trace(@src());
+ defer trace.end();
+
+ const module = try t.addRootInst(.{
+ .tag = .module,
+ .data = undefined,
+ });
+
+ const decl = BlockContext.init(module_node);
+
+ try t.pushScope();
+ defer t.popScope();
+
+ const statements = t.ast.nodeStatements(module_node);
+ for (statements) |statement| {
+ try t.previsitModuleStatement(statement);
+ }
+
+ const s = t.scope();
+
+ const temp = try t.gpa.alloc(Inst.Index, s.symbols.count() + 1);
+ defer t.gpa.free(temp);
+
+ {
+ // prevent concurrent access
+ s.symbols.lockPointers();
+ defer s.symbols.unlockPointers();
+ for (s.symbols.values(), 0..) |value, i| {
+ temp[i] = try t.transformFunction(decl, value.node);
+ }
+ }
+
+ const top_level_fn = try t.addRootInst(.{
+ .tag = .declaration,
+ .data = undefined,
+ });
+
+ temp[temp.len - 1] = top_level_fn;
+
+ const declarations: IndexList = .{
+ .start = @intCast(t.extra.list.items.len),
+ .len = @intCast(temp.len),
+ };
+ try t.extra.list.appendSlice(t.gpa, @ptrCast(temp));
+
+ const block = try t.addRootInst(.{
+ .tag = .block,
+ .data = undefined,
+ });
+
+ {
+ const scratch_top = t.scratch.items.len;
+ defer t.scratch.shrinkRetainingCapacity(scratch_top);
+
+ for (statements) |statement| {
+ try t.transformModuleStatement(decl, statement);
+ }
+
+ t.getInstData(block).* = EncodedData.encode(.block, .{
+ .src_node = decl.nodeOffset(module_node),
+ .extra_index = try t.extra.append(t.gpa, IndexList, try t.scratchToIndexList(scratch_top)),
+ });
+ }
+
+ const decl_extra, var trail = try t.extra.appendTrailing(t.gpa, Inst.Decl, .{
+ .kind = .function,
+ .name = NullTerminatedString.empty,
+ .hash = .{0} ** 16,
+ .instruction_count = @as(u32, @intCast(t.insts.len)) - @intFromEnum(top_level_fn),
+ }, dod.extraSizeOf(Inst.DeclFn));
+
+ trail.writeStruct(Inst.DeclFn, .{
+ .params = .{ .start = 0, .len = 0 },
+ .return_type = Inst.Ref.void_type,
+ .body = block.toRef(),
+ });
+
+ t.getInstData(top_level_fn).* = EncodedData.encode(.declaration, .{
+ .src_node = decl.nodeOffset(module_node),
+ .extra_index = decl_extra,
+ });
+
+ t.getInstData(module).* = EncodedData.encode(.module, .{
+ .src_node = 0,
+ .extra_index = try t.extra.append(t.gpa, IndexList, declarations),
+ });
+
+ return module;
+}
+
+fn previsitModuleStatement(t: *Transform, node: NodeIndex) !void {
+ const trace = tracer.trace(@src());
+ defer trace.end();
+
+ const tag = t.ast.nodes.items(.tag)[node];
+ switch (tag) {
+ .fn_decl => {
+ const fn_decl = t.ast.fullFnDecl(node);
+ const gop = try t.scope().symbols.getOrPut(t.gpa, fn_decl.id);
+ if (gop.found_existing) {
+ return error.DuplicateFunction;
+ }
+ gop.value_ptr.* = .{
+ .node = node,
+ .data = .declaration,
+ };
+ },
+ else => {},
+ }
+}
+
+fn transformModuleStatement(t: *Transform, decl: BlockContext, node: NodeIndex) !void {
+ const trace = tracer.trace(@src());
+ defer trace.end();
+
+ const tag = t.ast.nodes.items(.tag)[node];
+ switch (tag) {
+ .root,
+ .param_decl,
+ => unreachable,
+
+ .fn_decl => {}, // functions analyzed separately
+
+ else => _ = try t.transformBlockStatement(decl, node, false),
+ }
+}
+
+fn transformFunction(t: *Transform, outer_decl: BlockContext, node: NodeIndex) !Inst.Index {
+ const trace = tracer.trace(@src());
+ defer trace.end();
+
+ const decl = outer_decl.declOffset(node);
+ const fn_decl = t.ast.fullFnDecl(node);
+
+ const func = try t.addRootInst(.{
+ .tag = .declaration,
+ .data = undefined,
+ });
+
+ try t.pushScope();
+ defer t.popScope();
+
+ const param_list = brk: {
+ const scratch_top = t.scratch.items.len;
+ defer t.scratch.shrinkRetainingCapacity(scratch_top);
+
+ for (fn_decl.params) |param| {
+ try t.addScratch(Inst.Ref, try t.transformParamDecl(decl, param));
+ }
+
+ break :brk try t.scratchToRefList(scratch_top);
+ };
+
+ const return_type = if (fn_decl.return_type) |return_type_node|
+ try t.transformExpressionToBlock(decl, return_type_node, false)
+ else
+ Inst.Ref.void_type;
+
+ const body = try t.transformExpressionToBlock(decl, fn_decl.body, true);
+
+ const decl_extra = try t.extra.append(t.gpa, Inst.Decl, .{
+ .kind = .function,
+ .name = try t.addNullTerminatedString(fn_decl.id),
+ .hash = .{0} ** 16,
+ .instruction_count = @as(u32, @intCast(t.insts.len)) - @intFromEnum(func),
+ });
+
+ _ = try t.extra.append(t.gpa, Inst.DeclFn, .{
+ .params = param_list,
+ .return_type = return_type,
+ .body = body,
+ });
+
+ t.getInstData(func).* = EncodedData.encode(.declaration, .{
+ .src_node = 0,
+ .extra_index = decl_extra,
+ });
+
+ return func;
+}
+
+const TransformFunctionCtx = struct {
+ param_list: RefList,
+};
+
+/// Converts expressions into blocks with a break. This is important
+/// for places where a block is not allowed, such as a parameter list.
+/// This is also used for the function body, even though it is a block.
+/// Simple refs turn into just the ref
+fn transformExpressionToBlock(
+ t: *Transform,
+ decl: BlockContext,
+ node: NodeIndex,
+ comptime is_function: bool,
+) anyerror!Inst.Ref {
+ const trace = tracer.trace(@src());
+ defer trace.end();
+
+ const node_tag = t.ast.nodes.items(.tag)[node];
+ switch (node_tag) {
+ .id => {
+ // Lower builtin ids without using a block
+ const id = t.ast.basicNodeSlice(node);
+ if (BuiltInIds.get(id)) |ref| {
+ return ref;
+ }
+ },
+
+ else => {},
+ }
+
+ var block: ?Inst.Index = null;
+
+ const scratch_top = t.scratch.items.len;
+ defer t.scratch.shrinkRetainingCapacity(scratch_top);
+
+ if (is_function) {
+ const s = t.scopes.getLast();
+ for (s.symbols.values(), 0..) |*param, i| {
+ assert(param.data == .param);
+ const ref = try t.addInstRef(.{
+ .tag = .param,
+ .data = EncodedData.encode(.param, .{
+ .src_node = decl.nodeOffset(node),
+ .data = @intCast(i),
+ }),
+ });
+ param.data = .{ .immutable = ref };
+ }
+ }
+
+ const expr = try t.transformExpression(
+ if (is_function)
+ decl.withReturn(&block)
+ else
+ decl,
+ node,
+ );
+
+ // if there are no new instructions, we dont have to create a block context
+ if (scratch_top == t.scratch.items.len) {
+ const result = expr orelse unreachable; // null means we added break or return
+
+ // TODO: evaluate if `if` blocks would be fine in sema with instruction refs
+ // with this extra if check here, we filter out non-blocks.
+ if (result.toIndex() == null) {
+ return result;
+ }
+ }
+
+ if (expr) |result| {
+ if (result.toIndex()) |index| {
+ if (t.insts.items(.tag)[@intFromEnum(index)] == .block) {
+ return result;
+ }
+ }
+ }
+
+ const ret_block = if (block) |b|
+ b
+ else
+ try t.addRootInst(.{
+ .tag = .block,
+ .data = undefined,
+ });
+
+ if (expr) |result| {
+ _ = try t.addInstRef(.{
+ .tag = .@"break",
+ .data = EncodedData.encode(.@"break", .{
+ .src_node = decl.nodeOffset(node),
+ .extra_index = try t.extra.append(t.gpa, Inst.Break, .{
+ .block = ret_block,
+ .operand = result,
+ }),
+ }),
+ });
+ }
+
+ const content = try t.scratchToIndexList(scratch_top);
+
+ t.getInstData(ret_block).* = EncodedData.encode(.block, .{
+ .src_node = decl.nodeOffset(node),
+ .extra_index = try t.extra.append(t.gpa, IndexList, content),
+ });
+
+ return ret_block.toRef();
+}
+
+fn transformParamDecl(t: *Transform, decl: BlockContext, param: NodeIndex) !Inst.Ref {
+ const trace = tracer.trace(@src());
+ defer trace.end();
+
+ const param_decl = t.ast.fullParamDecl(param);
+ const value = try t.transformExpressionToBlock(decl, param_decl.type, false);
+
+ // add `sym` into the scope so that the next param's type
+ // can use this param's value, if known at compile time.
+ const sym = try t.scope().symbols.getOrPut(t.gpa, param_decl.id);
+ if (sym.found_existing) {
+ return error.DuplicateParam;
+ }
+
+ sym.value_ptr.* = .{
+ .node = param,
+ .data = .param,
+ };
+
+ return value;
+}
+
+const Builtin = enum {
+ js,
+ import,
+ fetch,
+ compile_error,
+
+ const map = std.ComptimeStringMap(Builtin, .{
+ .{ "js", .js },
+ .{ "import", .import },
+ .{ "fetch", .fetch },
+ .{ "compileError", .compile_error },
+ });
+
+ fn builtin_js(t: *Transform, decl: BlockContext, args: []const NodeIndex) !?Inst.Ref {
+ const trace = tracer.trace(@src());
+ defer trace.end();
+
+ if (args.len != 1) {
+ return error.InvalidBuiltin;
+ }
+
+ const source = try t.transformExpression(decl, args[0]) orelse return null;
+
+ return try t.addInstRef(.{
+ .tag = .js,
+ .data = EncodedData.encode(.js, .{
+ .src_node = decl.nodeOffset(args[0]),
+ .operand = source,
+ }),
+ });
+ }
+};
+
+/// This is the primary transformation function: converting an AST expression into instructions.
+/// `null` represents a `return` or `break`, aka this branch of code is dead.
+fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) anyerror!?Inst.Ref {
+ const node_tag = t.ast.nodes.items(.tag)[node];
+
+ const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)});
+ defer trace.end();
+
+ switch (node_tag) {
+ .root,
+ .param_decl,
+ .var_decl,
+ .fn_decl,
+ => unreachable, // not an expression
+
+ .id => {
+ const id = t.ast.basicNodeSlice(node);
+ if (BuiltInIds.get(id)) |ref| {
+ return ref;
+ }
+
+ if (mem.eql(u8, id, "_")) {
+ return error.InvalidDiscard;
+ }
+
+ if (mem.startsWith(u8, id, "@")) {
+ @panic("TODO: string identifiers");
+ }
+
+ const lookup = t.lookupId(id) orelse {
+ std.debug.print("lookupId failed for {s}\n", .{id});
+ return error.UnknownIdentifier;
+ };
+
+ return switch (lookup.data) {
+ .declaration => try t.addInstRef(.{
+ .tag = .load_decl,
+ .data = EncodedData.encode(.load_decl, .{
+ .src_node = decl.nodeOffset(node),
+ .string = try t.addNullTerminatedString(id),
+ }),
+ }),
+ .mutable,
+ .immutable,
+ => |ref| ref,
+ .param => unreachable, //unresolved param
+ };
+ },
+
+ inline .number_int, .number_float => |tag| {
+ const text = t.ast.basicNodeSlice(node);
+ const value = switch (tag) {
+ .number_int => std.fmt.parseFloat(f64, text) catch @panic("TODO: write my own float parser"),
+ .number_float => std.fmt.parseFloat(f64, text) catch @panic("TODO: write my own float parser"),
+ else => comptime unreachable,
+ };
+ return try t.addInstRef(.{
+ .tag = .number,
+ .data = EncodedData.encode(.number, .{
+ .src_node = decl.nodeOffset(node),
+ .extra_index = try t.extra.append(t.gpa, Float, Float.encode(value)),
+ }),
+ });
+ },
+
+ .string => {
+ const str = t.ast.basicNodeSlice(node);
+ const sliced = try t.addSlicedString(str[1 .. str.len - 1]);
+ return try t.addInstRef(.{
+ .tag = .string,
+ .data = EncodedData.encode(.string, .{
+ .src_node = decl.nodeOffset(node),
+ .extra_index = try t.extra.append(t.gpa, SlicedString, sliced),
+ }),
+ });
+ },
+
+ .assign => {
+ const ast_bin = t.ast.nodeData(node).bin;
+ const lhs = try t.transformExpression(decl, ast_bin.lhs) orelse return null;
+ const rhs = try t.transformExpression(decl, ast_bin.rhs) orelse return null;
+ const assign_inst = try t.addInstRef(.{
+ .tag = .store,
+ .data = EncodedData.encode(.store, .{
+ .src_node = decl.nodeOffset(node),
+ .extra_index = try t.extra.append(t.gpa, Inst.Bin, .{
+ .lhs = lhs,
+ .rhs = rhs,
+ }),
+ }),
+ });
+ return assign_inst;
+ },
+
+ inline .add,
+ .sub,
+ .mul,
+ .div,
+ .rem,
+ .pow,
+ .@"and",
+ .@"or",
+ .bit_and,
+ .bit_or,
+ .bit_xor,
+ .shl,
+ .shr,
+ .eq,
+ .ne,
+ .lt,
+ .le,
+ .gt,
+ .ge,
+ .add_assign,
+ .sub_assign,
+ .mul_assign,
+ .div_assign,
+ .rem_assign,
+ .pow_assign,
+ .and_assign,
+ .or_assign,
+ .bit_and_assign,
+ .bit_or_assign,
+ .bit_xor_assign,
+ .shl_assign,
+ .shr_assign,
+ => |tag| {
+ // This code path implements binary operations, as well as the
+ // compound compute + assign operation.
+ const is_assign = comptime tag.isAssignBin();
+ const teg_tag = @field(
+ Teg.Inst.Tag,
+ if (is_assign)
+ @tagName(tag)[0 .. @tagName(tag).len - "_assign".len]
+ else
+ @tagName(tag),
+ );
+ const ast_bin = t.ast.nodeData(node).bin;
+ const lhs = try t.transformExpression(decl, ast_bin.lhs) orelse return null;
+ const rhs = try t.transformExpression(decl, ast_bin.rhs) orelse return null;
+
+ const compute_inst = try t.addInstRef(.{
+ .tag = teg_tag,
+ .data = EncodedData.encode(teg_tag, .{
+ .src_node = decl.nodeOffset(node),
+ .extra_index = try t.extra.append(t.gpa, Inst.Bin, .{
+ .lhs = lhs,
+ .rhs = rhs,
+ }),
+ }),
+ });
+
+ if (is_assign) {
+ const assign_inst = try t.addInstRef(.{
+ .tag = .store,
+ .data = EncodedData.encode(.store, .{
+ .src_node = decl.nodeOffset(node),
+ .extra_index = try t.extra.append(t.gpa, Inst.Bin, .{
+ .lhs = lhs,
+ .rhs = compute_inst,
+ }),
+ }),
+ });
+ return assign_inst;
+ }
+
+ return compute_inst;
+ },
+
+ inline .negate,
+ .bool_not,
+ .bit_not,
+ => |tag| {
+ const ast_un = t.ast.nodeData(node).un;
+ const teg_tag = @field(Teg.Inst.Tag, @tagName(tag));
+
+ const operand = try t.transformExpression(decl, ast_un.expr) orelse
+ return null;
+
+ return try t.addInstRef(.{
+ .tag = teg_tag,
+ .data = EncodedData.encode(teg_tag, .{
+ .src_node = decl.nodeOffset(node),
+ .operand = operand,
+ }),
+ });
+ },
+
+ .label => {
+ const expr = t.ast.nodeData(node).label.expr;
+ const label = t.ast.tokenSlice(t.ast.nodes.items(.main_token)[node]);
+
+ var target: BlockContext.BreakTarget = .{
+ .parent = decl.break_target,
+ .label = label,
+ .is_loop = false,
+ };
+ const inner_ctx = decl.withBreak(&target);
+
+ const scratch_top = t.scratch.items.len;
+
+ const inner_expr = try t.transformExpression(inner_ctx, expr);
+
+ // if there are no new instructions, we dont have to create a block context
+ if (scratch_top == t.scratch.items.len)
+ return inner_expr orelse unreachable; // null means we added break or return
+
+ if (target.block_index) |block| {
+ if (inner_expr) |result| {
+ _ = try t.addInstRef(.{
+ .tag = .@"break",
+ .data = EncodedData.encode(.@"break", .{
+ .src_node = decl.nodeOffset(node),
+ .extra_index = try t.extra.append(t.gpa, Inst.Break, .{
+ .block = block,
+ .operand = result,
+ }),
+ }),
+ });
+ }
+
+ const content = try t.scratchToIndexList(scratch_top);
+ t.scratch.shrinkRetainingCapacity(scratch_top);
+
+ t.getInstData(block).* = EncodedData.encode(.block, .{
+ .src_node = decl.nodeOffset(node),
+ .extra_index = try t.extra.append(t.gpa, IndexList, content),
+ });
+
+ try t.addScratch(Inst.Index, block);
+
+ return block.toRef();
+ } else {
+ std.debug.print("Warn, unused label {s}", .{label});
+ }
+
+ return inner_expr;
+ },
+
+ inline .block_void,
+ .block_implicit_return,
+ => |tag| {
+ const statements = t.ast.nodeStatements(node);
+
+ var last: Inst.Ref = .void_value;
+ for (statements, 1..) |statement, i| {
+ const is_last = i == statements.len;
+ last = try t.transformBlockStatement(
+ decl,
+ statement,
+ tag == .block_implicit_return and is_last,
+ ) orelse {
+ return null;
+ };
+ }
+
+ return if (tag == .block_implicit_return) last else .void_value;
+ },
+
+ .builtin_call,
+ => {
+ const builtin = Builtin.map.get(t.ast.basicNodeSlice(node)[1..]) orelse {
+ return error.InvalidBuiltin;
+ };
+
+ const base = t.ast.nodeData(node).list;
+ const args = base.slice(&t.ast);
+
+ return switch (builtin) {
+ .js => Builtin.builtin_js(t, decl, args),
+ // .import => Builtin.import(t, decl, args),
+ // .fetch => Builtin.fetch(t, decl, args),
+ // .compile_error => Builtin.compile_error(t, decl, args),
+ else => @panic("TODO"),
+ };
+ },
+
+ .call => {
+ const call = t.ast.nodeData(node).call;
+ const args = t.ast.extraData(call.extra, Ast.Call).args.slice(&t.ast);
+
+ const base = try t.transformExpression(decl, call.base) orelse return null;
+
+ const args_list: RefList = brk: {
+ const args_top = t.call_args.items.len;
+ defer t.call_args.shrinkRetainingCapacity(args_top);
+
+ for (args) |arg| {
+ try t.call_args.append(
+ t.gpa,
+ try t.transformExpression(decl, arg) orelse
+ return null,
+ );
+ }
+
+ const items = t.call_args.items[args_top..];
+ const extra_top = t.extra.list.items.len;
+ try t.extra.list.appendSlice(t.gpa, @ptrCast(items));
+
+ break :brk .{
+ .start = @intCast(extra_top),
+ .len = @intCast(items.len),
+ };
+ };
+
+ const call_inst = try t.addInstRef(.{
+ .tag = .call,
+ .data = EncodedData.encode(.call, .{
+ .src_node = decl.nodeOffset(node),
+ .extra_index = try t.extra.append(t.gpa, Inst.Call, .{
+ .base = base,
+ .args = args_list,
+ }),
+ }),
+ });
+
+ return call_inst;
+ },
+
+ .field => {
+ const field = t.ast.nodeData(node).field;
+
+ const base = try t.transformExpression(decl, field.base) orelse return null;
+
+ // TODO: handle identifier escapes
+ const id = t.ast.tokenSlice(field.field_token);
+ const prop_name_str = try t.addSlicedString(id);
+
+ const string_ref = try t.addInstRef(.{
+ .tag = .string,
+ .data = EncodedData.encode(.string, .{
+ .src_node = decl.nodeOffset(node), // TODO: this is incorrect, there is no node
+ .extra_index = try t.extra.append(t.gpa, SlicedString, prop_name_str),
+ }),
+ });
+
+ return try t.addInstRef(.{
+ .tag = .field,
+ .data = EncodedData.encode(.field, .{
+ .src_node = decl.nodeOffset(node),
+ .extra_index = try t.extra.append(t.gpa, Inst.Bin, .{
+ .lhs = base,
+ .rhs = string_ref,
+ }),
+ }),
+ });
+ },
+
+ .@"if" => {
+ const if_node = t.ast.nodeData(node).@"if";
+ const if_data = t.ast.extraData(if_node.extra, Ast.If);
+
+ const cond = try t.transformExpression(decl, if_node.cond) orelse
+ return null;
+
+ const yes = try t.transformExpressionToBlock(
+ decl,
+ if_data.yes,
+ false,
+ );
+ const no = if (if_data.no.unwrap()) |no|
+ try t.transformExpressionToBlock(
+ decl,
+ no,
+ false,
+ )
+ else
+ Inst.Ref.void_value;
+
+ if (yes == .void_value and no == .void_value) {
+ // when both branches do nothing, we can omit the branch,
+ // however we still need to type-check the condition operand
+ _ = try t.addInstRef(.{
+ .tag = .ensure_bool,
+ .data = EncodedData.encode(.ensure_bool, .{
+ .src_node = decl.nodeOffset(node),
+ .operand = cond,
+ }),
+ });
+ return .void_value;
+ }
+
+ return try t.addInstRef(.{
+ .tag = .@"if",
+ .data = EncodedData.encode(.@"if", .{
+ .src_node = decl.nodeOffset(node),
+ .extra_index = try t.extra.append(t.gpa, Inst.If, .{
+ .cond = cond,
+ .yes = yes,
+ .no = no,
+ }),
+ }),
+ });
+ },
+
+ .@"return" => {
+ const ret_node = t.ast.nodeData(node).@"return";
+
+ const ret_block_ptr = decl.return_target orelse {
+ return error.InvalidReturn;
+ };
+
+ const ret_block = if (ret_block_ptr.*) |block|
+ block
+ else append_block: {
+ const block = try t.addRootInst(.{
+ .tag = .block,
+ .data = undefined,
+ });
+ ret_block_ptr.* = block;
+ break :append_block block;
+ };
+
+ const operand: Inst.Ref = if (ret_node.expr.unwrap()) |expr|
+ try t.transformExpression(decl.withReturn(null), expr) orelse
+ return null
+ else
+ .void_value;
+
+ _ = try t.addInstRef(.{
+ .tag = .@"break",
+ .data = EncodedData.encode(.@"break", .{
+ .src_node = decl.nodeOffset(node),
+ .extra_index = try t.extra.append(t.gpa, Inst.Break, .{
+ .block = ret_block,
+ .operand = operand,
+ }),
+ }),
+ });
+
+ return null; // control flow ends
+ },
+
+ .@"break" => {
+ const brk_node = t.ast.nodeData(node).@"break";
+ const label = t.ast.tokenSlice(
+ brk_node.label.unwrap() orelse
+ @panic("TODO: unlabelled breaks"),
+ );
+
+ const block = try decl.findAndMarkBreak(t, label) orelse {
+ return error.UnknownBreakTarget;
+ };
+
+ const operand: Inst.Ref = if (brk_node.value.unwrap()) |expr|
+ try t.transformExpression(decl.withReturn(null), expr) orelse
+ return null
+ else
+ .void_value;
+
+ _ = try t.addInstRef(.{
+ .tag = .@"break",
+ .data = EncodedData.encode(.@"break", .{
+ .src_node = decl.nodeOffset(node),
+ .extra_index = try t.extra.append(t.gpa, Inst.Break, .{
+ .block = block,
+ .operand = operand,
+ }),
+ }),
+ });
+
+ return null; // control flow ends
+ },
+
+ else => |tag| std.debug.panic("TODO: expr {s}", .{@tagName(tag)}),
+ }
+}
+
+fn transformBlockStatement(
+ t: *Transform,
+ decl: BlockContext,
+ node: NodeIndex,
+ is_implicit_return_expression: bool,
+) !?Inst.Ref {
+ const node_tag = t.ast.nodes.items(.tag)[node];
+
+ const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)});
+ defer trace.end();
+
+ switch (node_tag) {
+ .root => unreachable, // cannot appear in a block
+ .param_decl => unreachable, // cannot appear in a block
+
+ .var_decl => {
+ const var_decl = t.ast.fullVarDecl(node);
+ const type_ref = if (var_decl.type) |data_type|
+ try t.transformExpression(decl, data_type)
+ else
+ null;
+ const data_ref = try t.transformExpression(decl, var_decl.init) orelse
+ return null;
+ const data_coerced_ref = if (type_ref) |ref|
+ try t.addInstRef(.{
+ .tag = .as,
+ .data = EncodedData.encode(.as, .{
+ .src_node = decl.nodeOffset(node),
+ .extra_index = try t.extra.append(t.gpa, Inst.As, .{
+ .type = ref,
+ .operand = data_ref,
+ }),
+ }),
+ })
+ else
+ data_ref;
+
+ const mem_ref = if (var_decl.is_const)
+ data_coerced_ref
+ else
+ try t.addInstRef(.{
+ .tag = .memory,
+ .data = EncodedData.encode(.memory, .{
+ .src_node = decl.nodeOffset(node),
+ .operand = data_coerced_ref,
+ }),
+ });
+
+ try t.scope().symbols.put(t.gpa, var_decl.id, .{
+ .node = node,
+ .data = switch (var_decl.is_const) {
+ true => .{ .immutable = mem_ref },
+ false => .{ .mutable = mem_ref },
+ },
+ });
+
+ return mem_ref;
+ },
+
+ .fn_decl,
+ => |tag| {
+ std.debug.panic("TODO: block statement {s}", .{@tagName(tag)});
+ },
+
+ .assign,
+ => {
+ const bin = t.ast.nodeData(node).bin;
+ const lhs = bin.lhs;
+
+ if (t.ast.nodeTag(lhs) == .id and
+ mem.eql(u8, t.ast.basicNodeSlice(lhs), "_"))
+ {
+ // discard, instead of appending an .ensure_void check,
+ // we will add a .discard instruction instead.
+ // this will emit a warning if given void.
+ const result = try t.transformExpression(decl, bin.rhs) orelse
+ return null;
+ return try t.addInstRef(.{
+ .tag = .discard,
+ .data = EncodedData.encode(.discard, .{
+ .src_node = decl.nodeOffset(node),
+ .operand = result,
+ }),
+ });
+ } else {
+ // process this assignment as usual, do not append .ensure_void
+ return try t.transformExpression(decl, node);
+ }
+ },
+
+ .add_assign,
+ .sub_assign,
+ .mul_assign,
+ .div_assign,
+ .rem_assign,
+ .pow_assign,
+ .and_assign,
+ .or_assign,
+ .bit_and_assign,
+ .bit_or_assign,
+ .bit_xor_assign,
+ .shl_assign,
+ .shr_assign,
+ => {
+ // do not append a .ensure_void check
+ return try t.transformExpression(decl, node);
+ },
+
+ else => {
+ const result = try t.transformExpression(decl, node) orelse
+ return null;
+
+ if (!is_implicit_return_expression and result != .void_value) {
+ return try t.addInstRef(.{
+ .tag = .ensure_void,
+ .data = EncodedData.encode(.ensure_void, .{
+ .src_node = decl.nodeOffset(node),
+ .operand = result,
+ }),
+ });
+ }
+
+ return result;
+ },
+ }
+}
+
+// ---
+
+fn addRootInst(t: *Transform, inst: Inst) !Inst.Index {
+ try t.insts.append(t.gpa, inst);
+ return @enumFromInt(t.insts.len - 1);
+}
+
+fn addInstRef(t: *Transform, inst: Inst) !Inst.Ref {
+ try t.scratch.ensureUnusedCapacity(t.gpa, 1);
+ try t.insts.append(t.gpa, inst);
+ const index: Inst.Index = @enumFromInt(t.insts.len - 1);
+ t.scratch.appendAssumeCapacity(@intFromEnum(index));
+ return index.toRef();
+}
+
+fn pushScope(t: *Transform) !void {
+ try t.scopes.append(t.gpa, .{});
+}
+
+fn popScope(t: *Transform) void {
+ t.scope().deinit(t.gpa);
+ t.scopes.items[t.scopes.items.len - 1] = undefined;
+ t.scopes.items.len -= 1;
+}
+
+fn scope(t: *Transform) *Scope {
+ return &t.scopes.items[t.scopes.items.len - 1];
+}
+
+fn getInstData(t: *Transform, index: Inst.Index) *EncodedData {
+ return &t.insts.items(.data)[@intFromEnum(index)];
+}
+
+fn addScratch(t: *Transform, comptime T: type, value: T) !void {
+ try t.scratch.append(t.gpa, switch (T) {
+ Inst.Ref, Inst.Index => @intFromEnum(value),
+ else => @compileError("addScratch not implemented for " ++ @typeName(T)),
+ });
+}
+
+fn scratchToRefList(t: *Transform, scratch_top: usize) !RefList {
+ const items = t.scratch.items[scratch_top..];
+ const extra_top = t.extra.list.items.len;
+ try t.extra.list.appendSlice(t.gpa, t.scratch.items[scratch_top..]);
+ return .{
+ .start = @intCast(extra_top),
+ .len = @intCast(items.len),
+ };
+}
+
+fn scratchToIndexList(t: *Transform, scratch_top: usize) !IndexList {
+ const items = t.scratch.items[scratch_top..];
+ const extra_top = t.extra.list.items.len;
+ try t.extra.list.appendSlice(t.gpa, items);
+ return .{
+ .start = @intCast(extra_top),
+ .len = @intCast(items.len),
+ };
+}
+
+fn lookupId(t: *Transform, id: []const u8) ?Scope.Symbol {
+ var i = t.scopes.items.len;
+ while (i > 0) {
+ i -= 1;
+ if (t.scopes.items[i].symbols.get(id)) |sym| {
+ return sym;
+ }
+ }
+ return null;
+}
+
+fn addNullTerminatedString(t: *Transform, str: []const u8) !NullTerminatedString {
+ if (str.len == 0) {
+ return .empty;
+ }
+ // loop through strings trying to find an existing match, and dedupe memory
+ var pos: usize = 0;
+ while (mem.indexOfScalarPos(u8, str, pos, 0)) |i| : (pos = i + 1) {
+ const start_index = i - str.len;
+ if (mem.eql(u8, t.string_bytes.items[start_index..][0..str.len], str)) {
+ const string: NullTerminatedString = @enumFromInt(pos);
+ assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
+ return string;
+ }
+ }
+
+ const string: NullTerminatedString = @enumFromInt(t.string_bytes.items.len);
+ try t.string_bytes.ensureUnusedCapacity(t.gpa, str.len + 1);
+ t.string_bytes.appendSliceAssumeCapacity(str);
+ t.string_bytes.appendAssumeCapacity(0);
+ assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
+ return string;
+}
+
+fn addSlicedString(t: *Transform, str: []const u8) !SlicedString {
+ if (str.len == 0) {
+ return .{ .start = 0, .len = 0 };
+ }
+
+ // loop through strings trying to find an existing match, and dedupe memory
+ var pos: usize = 0;
+ while (mem.indexOfScalarPos(u8, str, pos, 0)) |i| : (pos = i + 1) {
+ const start_index = i - str.len;
+ if (mem.eql(u8, t.string_bytes.items[start_index..][0..str.len], str)) {
+ const string: SlicedString = .{
+ .start = @intCast(start_index),
+ .len = @intCast(str.len),
+ };
+ assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
+ return string;
+ }
+ }
+
+ const string: SlicedString = .{
+ .start = @intCast(t.string_bytes.items.len),
+ .len = @intCast(str.len),
+ };
+ try t.string_bytes.ensureUnusedCapacity(t.gpa, str.len);
+ t.string_bytes.appendSliceAssumeCapacity(str);
+ assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
+ return string;
+}
+
+const Transform = @This();
+const std = @import("std");
+const mem = std.mem;
+const Allocator = mem.Allocator;
+const assert = std.debug.assert;
+
+const Teg = @import("Teg.zig");
+const Inst = Teg.Inst;
+const NullTerminatedString = Teg.NullTerminatedString;
+const SlicedString = Teg.SlicedString;
+const RefList = Teg.RefList;
+const IndexList = Teg.IndexList;
+const EncodedData = Teg.EncodedData;
+const ExtraIndex = Teg.ExtraIndex;
+
+const dod = @import("dod.zig");
+const ExtraStore = dod.ExtraStore(.teg, .@"var");
+const Float = dod.Float;
+
+const Tokenizer = @import("Tokenizer.zig");
+const Token = Tokenizer.Token;
+const ByteOffset = Tokenizer.ByteOffset;
+
+const Ast = @import("Ast.zig");
+const Unused = Ast.Unused;
+const Node = Ast.Node;
+const Data = Node.Data;
+const SubList = Ast.SubList;
+const NodeIndex = Ast.NodeIndex;
+const TokenIndex = Ast.TokenIndex;
+const OptionalNodeIndex = Ast.OptionalNodeIndex;
+const OptionalTokenIndex = Ast.OptionalTokenIndex;
+const OptionalExtraIndex = Ast.OptionalExtraIndex;
+const optionalTokenIndex = OptionalTokenIndex.init;
+const optionalNodeIndex = OptionalNodeIndex.init;
+const optionalExtraIndex = OptionalExtraIndex.init;
+
+const InternPool = @import("InternPool.zig");
+
+const tracer = @import("tracer");
diff --git a/src/compiler/Value.zig b/src/compiler/Value.zig
new file mode 100644
index 0000000000000000000000000000000000000000..86c9dfc091c7cf2ae7322a8f253d0b25af593167
--- /dev/null
+++ b/src/compiler/Value.zig
@@ -0,0 +1,89 @@
+ip_index: InternPool.Index,
+
+pub fn fromInterned(i: InternPool.Index) Value {
+ assert(i != .none);
+ return .{ .ip_index = i };
+}
+
+pub fn toIntern(val: Value) InternPool.Index {
+ assert(val.ip_index != .none);
+ return val.ip_index;
+}
+
+pub fn toKey(val: Value, ip: *const InternPool) InternPool.Key {
+ return val.toIntern().toKey(ip);
+}
+
+pub fn asBool(val: Value) i32 {
+ return switch (val.toIntern()) {
+ .bool_true => true,
+ .bool_false => false,
+ else => unreachable,
+ };
+}
+
+pub fn asString(val: Value, ip: *InternPool) InternPool.String.WithLen {
+ const index = val.toIntern();
+ return ip.items.items(.data)[@intFromEnum(index)].decode(
+ .string,
+ ip.items.items(.tag)[@intFromEnum(index)],
+ )
+ .get(ip);
+}
+
+pub fn @"type"(val: Value, ip: *InternPool) Value {
+ std.debug.print("typeOfValue {any}\n", .{val.toKey(ip)});
+ const v: InternPool.Index = switch (val.toKey(ip)) {
+ .simple_type, .type_function => .type_type,
+
+ .int32 => .int32_type,
+ .float => .float_type,
+ .string => .string_type,
+ .simple_value => |v| switch (v) {
+ .true, .false => .bool_type,
+ .void => .void_type,
+ },
+ .func => |f| f.ty,
+ };
+ return Value.fromInterned(v);
+}
+
+/// used for validating ensure_void instructions
+pub fn isEnsureVoid(val: Value, ip: *InternPool) bool {
+ const intern = val.toIntern();
+ const i = @intFromEnum(intern);
+
+ return switch (ip.items.items(.tag)[i]) {
+ .simple_type => switch (ip.items.items(.data)[i].decode(.simple_type, .simple_type)) {
+ .@"opaque", .void => true,
+ else => true,
+ },
+ else => false,
+ };
+}
+
+const Value = @This();
+
+const std = @import("std");
+const assert = std.debug.assert;
+
+pub const InternPool = @import("InternPool.zig");
+
+pub fn fmt(v: Value, ip: *InternPool) Format {
+ return .{ .value = v, .ip = ip };
+}
+
+pub const Format = struct {
+ value: Value,
+ ip: *InternPool,
+
+ pub fn format(
+ self: Format,
+ comptime unused_fmt_str: []const u8,
+ _: std.fmt.FormatOptions,
+ writer: anytype,
+ ) !void {
+ comptime assert(unused_fmt_str.len == 0);
+ try writer.print("{any}", .{self.value.toKey(self.ip)});
+ }
+};
diff --git a/src/compiler/dod.zig b/src/compiler/dod.zig
new file mode 100644
index 0000000000000000000000000000000000000000..e30dbb3fa197d8b8a09acb6860ab141219fb0a92
--- /dev/null
+++ b/src/compiler/dod.zig
@@ -0,0 +1,382 @@
+const extra = @This();
+
+pub const Mutability = enum { @"const", @"var" };
+
+fn setAsU32(items: []u32, comptime T: type, index: usize, payload: T) void {
+ const fields = std.meta.fields(@TypeOf(payload));
+ var i = index;
+ inline for (fields) |field| {
+ const value = @field(payload, field.name);
+ switch (field.type) {
+ u32 => items[i] = value,
+ i32 => items[i] = @bitCast(value),
+
+ [16]u8 => {
+ const u32s: [4]u32 = @bitCast(value);
+ items[i] = u32s[0];
+ items[i + 1] = u32s[1];
+ items[i + 2] = u32s[2];
+ items[i + 3] = u32s[3];
+ },
+
+ else => brk: {
+ const info = @typeInfo(field.type);
+ if (info == .Enum) {
+ items[i] = @intFromEnum(value);
+ break :brk;
+ }
+
+ const struct_data = info.Struct;
+ if (struct_data.layout == .@"packed") {
+ items[i] = @bitCast(value);
+ break :brk;
+ }
+
+ @call(.always_inline, setAsU32, .{ items, field.type, i, value });
+ },
+ }
+ i += extraSizeOf(field.type);
+ }
+}
+
+fn getAsU32(items: []const u32, comptime T: type, index: usize) T {
+ const fields = std.meta.fields(T);
+ var i = index;
+ var result: T = undefined;
+ inline for (fields) |field| {
+ @field(result, field.name) = switch (field.type) {
+ u32 => items[i],
+ i32 => @bitCast(items[i]),
+
+ [16]u8 => @bitCast(items[i..][0..4].*),
+
+ else => brk: {
+ const info = @typeInfo(field.type);
+ if (info == .Enum) {
+ break :brk @enumFromInt(items[i]);
+ }
+
+ const struct_data = info.Struct;
+ if (struct_data.layout == .@"packed") {
+ break :brk @bitCast(items[i]);
+ }
+
+ break :brk @call(
+ .always_inline,
+ getAsU32,
+ .{ items, field.type, i },
+ );
+ },
+ };
+ i += extraSizeOf(field.type);
+ }
+ return result;
+}
+
+pub const StoreKind = enum {
+ intern_pool,
+ teg,
+ air,
+ tfr,
+
+ pub fn resolveStoreConst(
+ comptime kind: StoreKind,
+ owner: anytype,
+ ) *const kind.ResolvedStore(@TypeOf(owner)) {
+ return &owner.extra;
+ }
+
+ pub fn resolveStoreMut(
+ comptime kind: StoreKind,
+ owner: anytype,
+ ) *kind.ResolvedStore(@TypeOf(owner)) {
+ return &owner.extra;
+ }
+
+ pub fn ResolvedStore(
+ comptime kind: StoreKind,
+ comptime Owner: type,
+ ) type {
+ const ptr = @typeInfo(Owner).Pointer;
+ assert(ptr.size == .One);
+ const mutability = brk: for (@typeInfo(ptr.child).Struct.fields) |field| {
+ if (std.mem.eql(u8, field.name, "extra")) {
+ break :brk field.type.mutability;
+ }
+ } else unreachable;
+ return ExtraStore(kind, mutability);
+ }
+};
+
+fn ExtraWrap(comptime kind: StoreKind) type {
+ return struct {
+ pub fn Index(comptime T: type) type {
+ return enum(u32) {
+ _,
+
+ // Reads the data from `store.extra` and returns it.
+ pub fn get(index: @This(), owner: anytype) T {
+ return kind.resolveStoreConst(owner).get(T, @intFromEnum(index));
+ }
+
+ /// Returns data with a trailing reader to read data stored immediately after
+ /// the main data structure. What is stored should be documented on `T`
+ pub fn getTrail(index: @This(), owner: anytype) struct {
+ T,
+ kind.ResolvedStore(@TypeOf(owner)).TrailReader,
+ } {
+ const advance = comptime extraSizeOf(T);
+ return .{
+ index.get(owner),
+ .{ .index = @intFromEnum(index) + advance },
+ };
+ }
+ };
+ }
+ };
+}
+
+pub fn EncodedData(comptime Tag: type, backing_type: type) type {
+ return enum(backing_type) {
+ _,
+
+ pub fn decode(data: @This(), comptime tag: Tag, proof: Tag) tag.Data() {
+ assert(proof == tag); // incorrect memory access
+ const Data = tag.Data();
+ const info = @typeInfo(Data);
+ if (info == .Enum) {
+ return @enumFromInt(@intFromEnum(data));
+ }
+ if (info == .Struct) {
+ assert(backing_type == u64);
+ var fields: [2]u32 = @bitCast(@intFromEnum(data));
+ return getAsU32(&fields, Data, 0);
+ }
+ return @bitCast(@intFromEnum(data));
+ }
+
+ pub fn encode(comptime tag: Tag, data: tag.Data()) @This() {
+ const Data = tag.Data();
+ const info = @typeInfo(Data);
+ if (info == .Enum) {
+ return @enumFromInt(@intFromEnum(data));
+ }
+ if (info == .Struct) {
+ assert(backing_type == u64);
+ var fields: [2]u32 = undefined;
+ setAsU32(&fields, Data, 0, data);
+ return @enumFromInt(@as(backing_type, @bitCast(fields)));
+ }
+ return @enumFromInt(@as(backing_type, @bitCast(data)));
+ }
+ };
+}
+
+/// The number of 'u32's this type consumes in `ip.extra`.
+pub fn extraSizeOf(comptime T: type) usize {
+ return switch (T) {
+ u32,
+ i32,
+ => return 1,
+
+ [16]u8 => return 4,
+
+ else => {
+ const info = @typeInfo(T);
+ if (info == .Enum) {
+ comptime assert(@sizeOf(info.Enum.tag_type) <= 32);
+ return 1;
+ }
+
+ const struct_data = info.Struct;
+ if (struct_data.layout == .@"packed") {
+ comptime assert(struct_data.backing_integer == u32);
+ return 1;
+ }
+
+ const fields = std.meta.fields(T);
+ var size: usize = 0;
+ inline for (fields) |field| {
+ size += extraSizeOf(field.type);
+ }
+ return size;
+ },
+ };
+}
+
+pub fn ExtraStore(comptime set_kind: StoreKind, comptime set_mutability: Mutability) type {
+ return struct {
+ list: List = .{},
+
+ pub const kind = set_kind;
+ pub const mutability = set_mutability;
+
+ const Store = @This();
+
+ const index_holder = ExtraWrap(kind);
+ pub const Index = index_holder.Index;
+
+ pub const List = switch (mutability) {
+ .@"const" => struct {
+ items: []const u32 = &.{},
+
+ pub fn deinit(slice: *@This(), gpa: Allocator) void {
+ gpa.free(slice.items);
+ }
+ },
+ .@"var" => std.ArrayListUnmanaged(u32),
+ };
+
+ pub fn deinit(store: *Store, gpa: Allocator) void {
+ store.list.deinit(gpa);
+ store.* = undefined;
+ }
+
+ pub const TrailReader = struct {
+ index: usize,
+
+ pub fn read(trail: *TrailReader, ip: *const Store, comptime T: type) T {
+ defer trail.index += comptime extraSizeOf(T);
+ return ip.get(T, trail.index);
+ }
+
+ pub fn readInternPoolIndexSlice(trail: *TrailReader, len: u32) @import("InternPool.zig").Index.Slice {
+ defer trail.index += len;
+ return .{
+ .start = @intCast(trail.index),
+ .len = len,
+ };
+ }
+ };
+
+ pub const TrailWriter = struct {
+ store: *Store,
+ leftover: usize,
+
+ pub fn init(store: *Store, len: usize) TrailWriter {
+ return .{ .store = store, .leftover = len };
+ }
+
+ pub fn end(w: *TrailWriter) void {
+ assert(w.leftover == 0);
+ }
+
+ pub fn writeU32(w: *TrailWriter, value: u32) void {
+ assert(w.leftover > 0);
+ w.ip.extra.appendAssumeCapacity(value);
+ w.leftover -= 1;
+ }
+
+ pub fn writeSlice(w: *TrailWriter, slice: anytype) void {
+ const Child = std.meta.Child(@TypeOf(slice));
+ const child = @typeInfo(Child);
+ comptime assert(Child == u32 or (child == .Enum and child.Enum.tag_type == u32));
+ assert(w.leftover >= slice.len);
+ w.store.list.appendSliceAssumeCapacity(@ptrCast(slice));
+ w.leftover -= slice.len;
+ }
+
+ pub fn writeStruct(w: *TrailWriter, comptime T: type, data: T) void {
+ _ = appendAssumeCapacity(w.store, T, data);
+ }
+ };
+
+ pub fn append(store: *Store, gpa: Allocator, comptime T: type, payload: T) Allocator.Error!Index(T) {
+ comptime assert(mutability == .@"var");
+ try store.list.ensureUnusedCapacity(gpa, extraSizeOf(@TypeOf(payload)));
+ return store.appendAssumeCapacity(T, payload);
+ }
+
+ pub fn appendAssumeCapacity(ip: *Store, comptime T: type, payload: T) Index(T) {
+ comptime assert(mutability == .@"var");
+ const extra_index: u32 = @intCast(ip.list.items.len);
+ ip.list.items.len += extraSizeOf(@TypeOf(payload));
+ ip.set(T, extra_index, payload);
+ return @enumFromInt(extra_index);
+ }
+
+ pub fn appendTrailing(
+ store: *Store,
+ gpa: Allocator,
+ comptime T: type,
+ payload: T,
+ trailing_len: usize,
+ ) Allocator.Error!struct { Index(T), TrailWriter } {
+ comptime assert(mutability == .@"var");
+ try store.list.ensureUnusedCapacity(gpa, extraSizeOf(@TypeOf(payload)) + trailing_len);
+ const extra_index: u32 = @intCast(store.list.items.len);
+ store.list.items.len += extraSizeOf(@TypeOf(payload));
+ store.set(T, extra_index, payload);
+ return .{
+ @enumFromInt(extra_index),
+ TrailWriter.init(store, trailing_len),
+ };
+ }
+
+ fn set(store: *Store, comptime T: type, index: usize, payload: T) void {
+ comptime assert(mutability == .@"var");
+ setAsU32(store.list.items, T, index, payload);
+ }
+
+ fn get(store: *const Store, comptime T: type, index: usize) T {
+ return getAsU32(store.list.items, T, index);
+ }
+
+ pub fn toOwnedStore(store: *Store, gpa: Allocator) !ExtraStore(set_kind, .@"const") {
+ comptime assert(mutability == .@"var");
+ return .{ .list = .{ .items = try store.list.toOwnedSlice(gpa) } };
+ }
+
+ pub fn constant(store: *const Store) ExtraStore(set_kind, .@"const") {
+ return .{ .list = .{ .items = store.list.items } };
+ }
+ };
+}
+
+pub const Float = struct {
+ low: u32,
+ high: u32,
+
+ pub const pure_nan = Float{ .low = 0x7f800000, .high = 0 };
+
+ pub fn purifyNaN(f: f64) f64 {
+ if (std.math.isNan(f)) {
+ return comptime Float.pure_nan.decode();
+ }
+ return f;
+ }
+
+ pub fn assertPure(f: f64) f64 {
+ if (std.math.isNan(f)) {
+ assert(Float.pure_nan.eql(encode(f)));
+ }
+ return f;
+ }
+
+ pub fn encode(decoded: f64) Float {
+ if (std.math.isNan(decoded)) {
+ return pure_nan;
+ }
+ const as_bytes: [2]u32 = @bitCast(decoded);
+ return .{
+ .low = as_bytes[0],
+ .high = as_bytes[1],
+ };
+ }
+
+ pub fn decode(encoded: Float) f64 {
+ const as_bytes: [2]u32 = .{
+ encoded.low,
+ encoded.high,
+ };
+ return @bitCast(as_bytes);
+ }
+
+ pub fn eql(a: Float, b: Float) bool {
+ return a.low == b.low and a.high == b.high;
+ }
+};
+
+const std = @import("std");
+const assert = std.debug.assert;
+const Allocator = std.mem.Allocator;
diff --git a/src/compiler/main.zig b/src/compiler/main.zig
new file mode 100644
index 0000000000000000000000000000000000000000..289424463fdfa691861b11e24e54f2354bc36ee5
--- /dev/null
+++ b/src/compiler/main.zig
@@ -0,0 +1,180 @@
+const std = @import("std");
+const Tokenizer = @import("Tokenizer.zig");
+const Parse = @import("Parse.zig");
+const Transform = @import("Transform.zig");
+const Comp = @import("Comp.zig");
+
+const T = @import("tracer");
+pub const tracer_impl = @field(T, "none");
+
+pub fn main() !void {
+ std.debug.print("\r\x1b[K", .{});
+
+ try T.init();
+ defer T.deinit();
+
+ try T.init_thread(null);
+ defer T.deinit_thread();
+
+ var gpa: std.heap.GeneralPurposeAllocator(.{
+ .stack_trace_frames = 100,
+ }) = .{};
+ defer _ = gpa.deinit();
+ const alloc = gpa.allocator();
+
+ var maybe_file_path: ?[]const u8 = null;
+ defer if (maybe_file_path) |path| alloc.free(path);
+
+ var cwd_buf: [std.fs.max_path_bytes]u8 = undefined;
+ const cwd = try std.posix.getcwd(&cwd_buf);
+
+ var exe_name: []const u8 = undefined;
+
+ {
+ var args = try std.process.argsWithAllocator(alloc);
+ defer args.deinit();
+
+ exe_name = try alloc.dupe(u8, std.fs.path.basename(args.next() orelse "cli"));
+ errdefer alloc.free(exe_name);
+
+ while (args.next()) |arg| {
+ maybe_file_path = try std.fs.path.resolve(alloc, &.{ cwd, arg });
+ break;
+ }
+ }
+ defer alloc.free(exe_name);
+
+ const file_path = maybe_file_path orelse {
+ std.debug.print("Usage: {s} \n", .{exe_name});
+ std.process.exit(1);
+ };
+
+ var comp: Comp = .{ .gpa = T.tracingAllocator(&alloc) };
+ try comp.init();
+ defer comp.deinit();
+ const main_file = try comp.getFile(file_path);
+
+ const teg = try main_file.getTeg(comp.gpa);
+ {
+ const stderr = std.io.getStdErr().writer();
+ try teg.dumpToWriter(stderr);
+ }
+
+ try comp.semaEntryPoint(main_file);
+
+ const output = std.io.getStdOut().writer();
+ var bufw = std.io.bufferedWriter(output);
+ const writer = bufw.writer();
+ try comp.link(writer);
+ try bufw.flush();
+
+ // var error_pos: u32 = 0;
+ // var tokens = Tokenizer.tokenize(source, alloc.allocator(), &error_pos) catch |err| {
+ // var line: usize = 1;
+ // var byte: usize = 0;
+ // var line_data: []const u8 = undefined;
+ // while (true) {
+ // const b = std.mem.indexOfScalarPos(u8, source, byte, '\n') orelse {
+ // line_data = source[byte..];
+ // break;
+ // };
+ // if (b > error_pos) {
+ // line_data = source[byte..b];
+ // break;
+ // }
+ // byte = b + 1;
+ // line += 1;
+ // }
+ // var col: usize = 1;
+ // var arrow_pos: usize = 0;
+ // while (byte < error_pos) {
+ // const b = source[byte];
+ // std.debug.assert(b != '\n');
+ // byte += std.unicode.utf8ByteSequenceLength(b) catch break;
+ // col += 1;
+
+ // arrow_pos += 1; // TODO: grapheme cluster
+ // }
+
+ // const stderr = std.io.getStdErr();
+ // const config = std.io.tty.detectConfig(stderr);
+ // var buf = std.io.bufferedWriter(stderr.writer());
+ // const writer = buf.writer();
+
+ // try config.setColor(writer, .red);
+ // try writer.writeAll("error");
+ // try config.setColor(writer, .bright_black);
+ // try writer.writeAll(": ");
+ // try config.setColor(writer, .reset);
+ // switch (err) {
+ // error.OutOfMemory => try writer.writeAll("Out of Memory"),
+ // error.InvalidUtf8 => try writer.writeAll("Invalid UTF-8"),
+ // error.InvalidSyntax => try writer.writeAll("Syntax Error"),
+ // error.TrailingNumericSeparator => try writer.writeAll("Trailing Numeric Separator"),
+ // error.UnterminatedString => try writer.writeAll("Unterminated String"),
+ // error.InvalidEscapeSequence => try writer.writeAll("Invalid Escape Sequence"),
+ // error.InvalidEscapeCodepoint => try writer.writeAll("Invalid Escape Codepoint"),
+ // error.GitConflict => try writer.writeAll("Found Git Conflict"),
+ // }
+ // try writer.print(" at {d}:{d}\n ", .{ line, col });
+ // try writer.writeAll(line_data);
+ // try writer.writeAll("\n ");
+ // try writer.writeByteNTimes(' ', col);
+
+ // try config.setColor(writer, .bright_green);
+ // try writer.writeAll("^");
+ // try config.setColor(writer, .reset);
+ // try writer.writeAll("\n");
+
+ // try buf.flush();
+ // return err;
+ // };
+ // defer tokens.deinit(alloc.allocator());
+
+ // // std.debug.print("tokens.len = {d}\n", .{tokens.len});
+ // // std.debug.print("token list bytes = {d}\n", .{tokens.toMultiArrayList().allocatedBytes().len});
+ // // std.debug.print("source bytes = {d}\n", .{source.len});
+ // // for (
+ // // tokens.items(.tag),
+ // // tokens.items(.offset),
+ // // ) |tag, offset| {
+ // // std.debug.print(".tag = {s} .offset = {d}\n", .{
+ // // @tagName(tag),
+ // // offset,
+ // // });
+ // // }
+
+ // const ast = ast: {
+ // var p = Parse.init(alloc.allocator(), source, tokens);
+ // errdefer p.deinit();
+ // try p.nodes.ensureUnusedCapacity(alloc.allocator(), 100);
+ // p.parseRoot() catch |err| {
+ // const tok = p.fullToken(p.pos);
+
+ // std.debug.print("ast_error = {?}\n", .{p.ast_error});
+
+ // std.debug.print("tok: {s} - '{s}' - {d}\n", .{
+ // @tagName(tok.tag),
+ // tok.slice(p.source),
+ // tok.offset,
+ // });
+
+ // return err;
+ // };
+ // break :ast try p.finalize(tokens);
+ // };
+
+ // try ast.dumpToWriter(std.io.getStdOut().writer());
+
+ // const teg = try Transform.transform(ast, alloc.allocator());
+ // std.debug.print("teg.insts.len = {d}\n", .{teg.insts.len});
+ // std.debug.print("teg.string_bytes.len = {d}\n", .{teg.string_bytes.len});
+ // std.debug.print("teg.extra.len = {d}\n", .{teg.extra.len});
+
+ // for (teg.insts.items(.tag), teg.insts.items(.data)) |tag, data| {
+ // std.debug.print("{s} - {}\n", .{
+ // @tagName(tag),
+ // data,
+ // });
+ // }
+}
diff --git a/src/compiler/unicode_props.zig b/src/compiler/unicode_props.zig
new file mode 100644
index 0000000000000000000000000000000000000000..f7e2f98f9686de6527a169bb37435880a7a890e9
--- /dev/null
+++ b/src/compiler/unicode_props.zig
@@ -0,0 +1,103 @@
+//! Wrapper around the 'zg' PropsData library.
+const std = @import("std");
+const PropsData = @import("zg:PropsData");
+const T = @import("tracer");
+
+var props: ?PropsData = undefined;
+var props_memory: []u8 = undefined;
+
+const raw_allocator = if (@import("builtin").target.isWasm())
+ std.heap.wasm_allocator
+else
+ std.heap.page_allocator;
+
+pub fn init() !void {
+ if (props != null) return;
+
+ const trace = T.trace(@src());
+ defer trace.end();
+
+ // We know exactly how much this uses, so let's preallocate these pages and use a FBA.
+ const allocation_size = 8704 + 34560 + 8704 + 17664 + 8704 + 25856;
+
+ if (allocation_size == 0) {
+ var logging_alloc = std.heap.loggingAllocator(std.heap.page_allocator);
+ props = try PropsData.init(logging_alloc.allocator());
+ std.log.info("please write the above numbers into unicode.zig to prealloc this", .{});
+ std.process.exit(1);
+ }
+
+ props_memory = try raw_allocator.alloc(u8, allocation_size);
+ errdefer std.heap.page_allocator.free(props_memory);
+
+ var fba = std.heap.FixedBufferAllocator.init(props_memory);
+ props = try PropsData.init(fba.allocator());
+ std.debug.assert(fba.end_index == allocation_size);
+}
+
+pub fn deinit() void {
+ std.debug.assert(props != null);
+ raw_allocator.free(props_memory);
+ props = null;
+ props_memory = undefined;
+}
+
+/// True if `cp` is a mathematical symbol.
+pub inline fn isMath(cp: u21) bool {
+ return props.?.isMath(cp);
+}
+
+/// True if `cp` is an alphabetic character.
+pub inline fn isAlphabetic(cp: u21) bool {
+ return props.?.isAlphabetic(cp);
+}
+
+/// True if `cp` is a valid identifier start character.
+pub inline fn isIdStart(cp: u21) bool {
+ return props.?.isIdStart(cp);
+}
+
+/// True if `cp` is a valid identifier continuation character.
+pub inline fn isIdContinue(cp: u21) bool {
+ return props.?.isIdContinue(cp);
+}
+
+/// True if `cp` is a valid extended identifier start character.
+pub inline fn isXidStart(cp: u21) bool {
+ return props.?.isXidStart(cp);
+}
+
+/// True if `cp` is a valid extended identifier continuation character.
+pub inline fn isXidContinue(cp: u21) bool {
+ return props.?.isXidContinue(cp);
+}
+
+/// True if `cp` is a whitespace character.
+pub inline fn isWhitespace(cp: u21) bool {
+ return props.?.isWhitespace(cp);
+}
+
+/// True if `cp` is a hexadecimal digit.
+pub inline fn isHexDigit(cp: u21) bool {
+ return props.?.isHexDigit(cp);
+}
+
+/// True if `cp` is a diacritic mark.
+pub inline fn isDiacritic(cp: u21) bool {
+ return props.?.isDiacritic(cp);
+}
+
+/// True if `cp` is numeric.
+pub inline fn isNumeric(cp: u21) bool {
+ return props.?.isNumeric(cp);
+}
+
+/// True if `cp` is a digit.
+pub inline fn isDigit(cp: u21) bool {
+ return props.?.isDigit(cp);
+}
+
+/// True if `cp` is decimal.
+pub inline fn isDecimal(cp: u21) bool {
+ return props.?.isDecimal(cp);
+}
diff --git a/src/compiler/util.zig b/src/compiler/util.zig
new file mode 100644
index 0000000000000000000000000000000000000000..2896fe9d2c8b08afd599693f52eb05f50597e4b4
--- /dev/null
+++ b/src/compiler/util.zig
@@ -0,0 +1,20 @@
+/// TODO: move to io utility file
+pub fn detectTTYFromWriter(writer: anytype) std.io.tty.Config {
+ if (@hasField(@TypeOf(writer), "unbuffered_writer")) {
+ return detectTTYFromWriter(writer.unbuffered_writer);
+ }
+ if (@TypeOf(writer) == std.fs.File.Writer) {
+ return std.io.tty.detectConfig(writer.context);
+ }
+ if (@TypeOf(writer) == std.io.AnyWriter) {
+ const writeFn = comptime (std.fs.File{ .handle = undefined })
+ .writer().any().writeFn;
+ if (writer.writeFn == writeFn) {
+ const file: *const std.fs.File = @alignCast(@ptrCast(writer.context));
+ return std.io.tty.detectConfig(file.*);
+ }
+ }
+ return .no_color;
+}
+
+const std = @import("std");
diff --git a/src/wasm/enum_codegen.zig b/src/wasm/enum_codegen.zig
new file mode 100644
index 0000000000000000000000000000000000000000..7e623e9ffb09bc0113fcb6fbf716787dc4ac8b8f
--- /dev/null
+++ b/src/wasm/enum_codegen.zig
@@ -0,0 +1,28 @@
+const std = @import("std");
+const Tokenizer = @import("Tokenizer.zig");
+const Ast = @import("Ast.zig");
+const Teg = @import("Teg.zig");
+// const InternPool = @import("InternPool.zig");
+
+pub fn main() !void {
+ var buf = std.io.bufferedWriter(std.io.getStdOut().writer());
+ defer buf.flush() catch {};
+ const w = buf.writer();
+
+ inline for (.{
+ .{ "TokenTag", Tokenizer.Token.Tag },
+ .{ "NodeTag", Ast.Node.Tag },
+ .{ "TegInstTag", Teg.Inst.Tag },
+ .{ "TegInstRef", Teg.Inst.Ref },
+ // .{ "InternPoolIndex", InternPool.Index },
+ }) |decl| {
+ const name, const enum_type = decl;
+
+ try w.print("export const {s} = {{\n", .{name});
+ for (std.enums.values(enum_type)) |tag| {
+ try w.print(" {d}: \"{s}\",\n", .{ @intFromEnum(tag), @tagName(tag) });
+ }
+ try w.writeAll("};\n");
+ try w.writeAll("\n");
+ }
+}
diff --git a/src/wasm/wasm_api.zig b/src/wasm/wasm_api.zig
new file mode 100644
index 0000000000000000000000000000000000000000..441bdb5d06b05667fc79ad33d5d4ca8e3e7423fe
--- /dev/null
+++ b/src/wasm/wasm_api.zig
@@ -0,0 +1,32 @@
+//! A subset of the compiler is exposed to wasm
+//! This powers the playground
+
+extern fn iterateTokenCallback(
+ tag: i32,
+ offset: ByteOffset,
+) void;
+
+export fn iterateTokens(
+ source_bytes: [*]const u8,
+ source_len: u32,
+) callconv(.C) i32 {
+ unicode_props.init() catch @panic("unicode_props.init failed");
+
+ var tokenizer: Tokenizer = .{ .code = source_bytes[0..source_len], .pos = 0 };
+ tokenizer.skipWhitespace();
+ while (tokenizer.next() catch {
+ return @intCast(tokenizer.pos);
+ }) |token| {
+ iterateTokenCallback(@intFromEnum(token.tag), token.offset);
+ tokenizer.skipWhitespace();
+ }
+
+ return -1;
+}
+
+const std = @import("std");
+const wasm_allocator = std.heap.wasm_allocator;
+const Tokenizer = @import("Tokenizer.zig");
+const Token = Tokenizer.Token;
+const ByteOffset = Tokenizer.ByteOffset;
+const unicode_props = @import("unicode_props.zig");