Commit 1d2c9a5425cd1d9a4bbdc69cbf2bd7e842fd02b7 | ix

author	dave caruso <me@paperdave.net> 2024-06-01 21:21:51-07:00
committer	dave caruso <me@paperdave.net> 2024-06-01 21:21:51-07:00
log	1d2c9a5425cd1d9a4bbdc69cbf2bd7e842fd02b7
tree	cd200116572339279769b3171b5a258cd8ae3c12
parent	649aec323b3d81b3f921d90132a3ce548cd47d3b
signature	Commit is signed but in an unrecognized format.

implement binary operand stuff

17 files changed, 927 insertions(+), 705 deletions(-)

CHANGELOG.md
build
build.cmd
build.zig
demo.todo
doc
- language-manual.md
- language-reference.md
src
- core
- langref
  - generate-langref.zig
  - template.html
- manual
  - generate-man.zig
  - template.html

CHANGELOG.md+3

...	...	@@ -2,12 +2,15 @@
2	2
3	3	- feat(sema): coersion: numeric\_literal -> float
4	4	- feat(sema): prevent passing non-runtime compatible types like `type` to opaque functions
	5	- feat(sema): properly validate and coerce binary operations
	6	- feat(sema): string concat with `+`
5	7	- feat(transform): lhs of assignments now validated
6	8	- feat(transform): mark useless discard
7	9	- feat(transform): parse string escapes properly
8	10	- feat(link): print string escapes properly
9	11	- feat: implement the `comptime` keyword (many comptime ops are not implemented)
10	12	- feat: add `@jsNew` to create JavaScript `new` expressions
	13	- fix(parse): top-level if statements previously required semicolons
11	14	- fix(sema): properly coerce assignments
12	15	- fix(transform): empty block generated in top level code tripped sema assertion
13	16	- fix(link): emit parenthesis to preserve operator precedence

build+5-5

...	...	@@ -1,6 +1,6 @@
1	1	#!/bin/sh
2	2	# This script will build the compiler on any POSIX system.
3		# It automatically installs a Zig compiler to ./zig-cache
	3	# It automatically installs a Zig compiler to ./.zig-cache
4	4	# if needed. The output will be in ./zig-out/bin.
5	5	#
6	6	# `curl` and `tar` is required in order to download and
...	...	@@ -11,7 +11,7 @@ cd -- "$(dirname -- "$0")"
11	11	ZIG_VERSION=$(grep .minimum_zig_version build.zig.zon \
12	12	\| cut -d '"' -f 2)
13	13
14		ZIG="zig-cache/zig/$ZIG_VERSION/zig"
	14	ZIG=".zig-cache/zig/$ZIG_VERSION/zig"
15	15	if [ ! -f "$ZIG" ]; then
16	16	OS=$(uname -s \| tr '[:upper:]' '[:lower:]')
17	17	if [ "$OS" = "darwin" ]; then
...	...	@@ -40,12 +40,12 @@ if [ ! -f "$ZIG" ]; then
40	40	exit 1
41	41	fi
42	42
43		mkdir -p zig-cache/zig/$ZIG_VERSION
	43	mkdir -p .zig-cache/zig/$ZIG_VERSION
44	44	curl -fsSL --retry 3 --retry-delay 3 $URL \
45		\| tar -xJ --strip-components=1 -C zig-cache/zig/$ZIG_VERSION
	45	\| tar -xJ --strip-components=1 -C .zig-cache/zig/$ZIG_VERSION
46	46	if [ "$($ZIG version)" != "$ZIG_VERSION" ]; then
47	47	echo "Zig failed to install."
48		rm -rf zig-cache/zig/$ZIG_VERSION
	48	rm -rf .zig-cache/zig/$ZIG_VERSION
49	49	exit 1
50	50	fi
51	51	fi

build.cmd+9-9

...	...	@@ -1,7 +1,7 @@
1	1	@ECHO OFF
2	2	@REM This script will build the compiler on any Windows system
3		@REM It automatically installs a Zig compiler to ./zig-cache
4		@REM The output will be in ./zig-out/bin
	3	@REM It automatically installs a Zig compiler to \/.zig-cache
	4	@REM The output will be in .\zig-out\bin
5	5
6	6	SET ROOT=%~dp0
7	7	FOR /F "tokens=3 delims=, " %%i IN (
...	...	@@ -9,9 +9,9 @@ FOR /F "tokens=3 delims=, " %%i IN (
9	9	"%ROOT%build.zig.zon"'
10	10	) DO SET ZIG_VERSION=%%i
11	11	SET ZIG_VERSION=%ZIG_VERSION:~1,-1%
12		IF NOT EXIST "%ROOT%zig-cache\zig\%ZIG_VERSION%\zig.exe" GOTO :find_powershell
	12	IF NOT EXIST "%ROOT%.zig-cache\zig\%ZIG_VERSION%\zig.exe" GOTO :find_powershell
13	13	:build
14		"%ROOT%zig-cache\zig\%ZIG_VERSION%\zig.exe" build %*
	14	"%ROOT%.zig-cache\zig\%ZIG_VERSION%\zig.exe" build %*
15	15	EXIT /b %ERRORLEVEL%
16	16	:find_powershell
17	17	FOR %%i IN (pwsh powershell) DO WHERE %%i /Q && SET PS=%%i && GOTO :download
...	...	@@ -20,10 +20,10 @@ EXIT /b 1
20	20	:download
21	21	SET ZIG_ARCH=x86_64
22	22	SET URI=https://ziglang.org/builds/zig-windows-%ZIG_ARCH%-%ZIG_VERSION%.zip
23		SET X=$null = New-Item -ItemType Directory -Force -Path zig-cache\zig
24		SET X=%X%;Invoke-RestMethod -Uri %URI% -OutFile zig-cache/zig-%ZIG_VERSION%.zip
25		SET X=%X%;Expand-Archive zig-cache/zig-%ZIG_VERSION%.zip -DestinationPath zig-cache/zig
26		SET X=%X%;Remove-Item zig-cache/zig-%ZIG_VERSION%.zip
	23	SET X=$null = New-Item -ItemType Directory -Force -Path .zig-cache\zig
	24	SET X=%X%;Invoke-RestMethod -Uri %URI% -OutFile .zig-cache/zig-%ZIG_VERSION%.zip
	25	SET X=%X%;Expand-Archive .zig-cache/zig-%ZIG_VERSION%.zip -DestinationPath .zig-cache/zig
	26	SET X=%X%;Remove-Item .zig-cache/zig-%ZIG_VERSION%.zip
27	27	%PS% -NoProfile -Command "%X%"
28		MOVE zig-cache\zig\zig-windows-%ZIG_ARCH%-%ZIG_VERSION% zig-cache\zig\%ZIG_VERSION% >NUL
	28	MOVE .zig-cache\zig\zig-windows-%ZIG_ARCH%-%ZIG_VERSION% .zig-cache\zig\%ZIG_VERSION% >NUL
29	29	GOTO :build

build.zig+14-14

...	...	@@ -59,22 +59,22 @@ pub fn build(b: *std.Build) void {
59	59	}
60	60
61	61	// Language Reference Generator
62		const langref_generator = b.addExecutable(.{
63		.name = "generate-langref",
64		.root_source_file = b.path("src/langref/generate-langref.zig"),
	62	const manual_generator = b.addExecutable(.{
	63	.name = "generate-man",
	64	.root_source_file = b.path("src/manual/generate-man.zig"),
65	65	.target = native_target,
66	66	.optimize = .Debug,
67	67	.link_libc = true,
68	68	});
69		langref_generator.root_module.addImport("build_options", native_debug.opts_mod);
70		langref_generator.root_module.addImport("core", native_debug.core);
71		const langref_run = b.addRunArtifact(langref_generator);
72		langref_run.addFileArg(native_debug.compiler.getEmittedBin());
73		langref_run.addFileArg(b.path("doc/language-reference.md"));
74		langref_run.addFileArg(b.path("src/langref/template.html"));
75		const langref_out = langref_run.addOutputFileArg("language-reference.html");
76		const install_langref = b.addInstallFile(langref_out, "doc/README.html");
77		b.step("langref", "build language reference").dependOn(&install_langref.step);
	69	manual_generator.root_module.addImport("build_options", native_debug.opts_mod);
	70	manual_generator.root_module.addImport("core", native_debug.core);
	71	const manual_run = b.addRunArtifact(manual_generator);
	72	manual_run.addFileArg(native_debug.compiler.getEmittedBin());
	73	manual_run.addFileArg(b.path("doc/language-manual.md"));
	74	manual_run.addFileArg(b.path("src/manual/template.html"));
	75	const manual_out = manual_run.addOutputFileArg("language-reference.html");
	76	const install_manual = b.addInstallFile(manual_out, "doc/README.html");
	77	b.step("manual", "build language reference").dependOn(&install_manual.step);
78	78
79	79	// const wasm_api = b.addExecutable(.{
80	80	// .name = "compiler",
...	...	@@ -137,8 +137,8 @@ pub fn build(b: *std.Build) void {
137	137	).step);
138	138	}
139	139
140		package.dependOn(&b.addInstallFile(b.path("CHANGELOG.md"), "release/CHANGELOG.md").step);
141		package.dependOn(&b.addInstallFile(langref_out, "release/README.html").step);
	140	package.dependOn(&b.addInstallFile(b.path("CHANGELOG.md"), "package/CHANGELOG.md").step);
	141	package.dependOn(&b.addInstallFile(manual_out, "package/README.html").step);
142	142	}
143	143	}
144	144

demo.todo created+10

...	...	@@ -0,0 +1,10 @@
	1	fn something(a: int) -> bool {
	2	if a == 0 {
	3	return false;
	4	}
	5
	6	a > 100
	7	}
	8
	9	let print = @js("console").log;
	10	print(something(0xFF), something(0));

doc/language-manual.md created+160

...	...	@@ -0,0 +1,160 @@
	1	# New Language Project
	2
	3	This is a high-level, general purpose programming language.
	4
	5	Key goals of this project are:
	6
	7	- Ease of Use: Use of type inference keeps code concise and easy to read and write.
	8	- Fast Iteration: The compiler is designed for quick incremental compilation and hot module reloading. (note: incremental is not yet available)
	9	- TODO: some more
	10
	11	It primarily focuses on merging some great points of Rust, Zig, and JavaScript's language designs:
	12
	13	- Rust - Expressive control flow that is easy to write.
	14	- Zig - Execute any code at compile time, everything is an expression.
	15	- JavaScript - Garbage collected values with a versitile type system.
	16
	17	## Standard Library
	18
	19	Currently, there is no standard library. One will be introduced later.
	20
	21	## Hello World
	22
	23	```lang:run
	24	let print = @js("console").log;
	25	print("Hello, world!");
	26	```
	27
	28	The `print` function is implemented by getting an external reference to the `console` JavaScript global, and then retriving the `log` property from it.
	29
	30	## Primative Data Types
	31
	32	\| name \| ts equivilent \| description \|
	33	\| ---------- \| --------------------- \| ---------------------------------------------- \|
	34	\| `int` \| `number` \| a number in signed 32-bit range \|
	35	\| `float` \| `number` \| a 64-bit floating point number \|
	36	\| `bool` \| `boolean` \| either `true` or `false` \|
	37	\| `string` \| `string` \| a sequence of text characters \|
	38	\| `void` \| `undefined` or `null` \| detectable lack of a value \|
	39	\| `opaque` \| `unknown` or `any` \| could be any value \|
	40	\| `noreturn` \| `never` \| control flow prevents this value from existing \|
	41
	42	> note: `bigint` is planned but not added
	43
	44	## Built-in Constants
	45
	46	In addition to the above types being built-in constants, the following are available.
	47
	48	\| name \| description \|
	49	\| ------- \| ------------------------------------- \|
	50	\| `true` \| boolean true (result of `1 == 1`) \|
	51	\| `false` \| boolean false (result of `1 == 2`) \|
	52	\| `std` \| handle to the standard library module \|
	53
	54	These constants are reserved names, and it is an error to re-declare them.
	55
	56	```lang:build
	57	let true = 1 == 0;
	58	```
	59
	60	## Variables
	61
	62	There are two types of variables, `let` and `var`.
	63
	64	`let` bindings cannot be overridden, and should be thought of as aliases of the expression within. Referring to these identifiers is equivilent to inlining it's where it was used, aside from side effect duplication.
	65
	66	```lang:run
	67	let hello: int = 1;
	68	let world = hello + 2;
	69	let hello = hello * world;
	70
	71	@js("console").log(hello);
	72	```
	73
	74	`var` bindings create mutable memory locations at runtime. This means they can be re-assigned to:
	75
	76	```lang:run
	77	var message = "first message";
	78
	79	if @js("Math").random() > 0.5 {
	80	message = "second message";
	81	}
	82
	83	@js("console").log(message);
	84	```
	85
	86	Using `let` is preferred whenever possible, as it will lead to more readable code, and the compiler is able to optimize it better. The above can be rewritten as:
	87
	88	```lang:run
	89	let message = if @js("Math").random() > 0.5 {
	90	"second message"
	91	} else {
	92	"first message"
	93	};
	94
	95	@js("console").log(message);
	96	```
	97
	98	It should be noted that numeric literals are a comptime-only data type, meaning it cannot be assigned to a `var` without an explicit type annotation. Since `int` is a strict subset of valid `float`s, the coersion will implicitly be a float, but a compiler warning will be emitted.
	99
	100	```lang
	101	var number = 0; // `number` is a `float`, but not explicit
	102	var number: int = 0; // `number` is `int`
	103	var number: float = 0; // `number` is `float`
	104	```
	105
	106	## Functions
	107
	108	Functions are declared with the `fn` keyword. Parameter types must be annotated. The function body is a block, which may use implicit return to return.
	109
	110	```lang:run
	111	fn add(a: int, b: int) -> int {
	112	a + b
	113	}
	114
	115	let print = @js("console").log;
	116	print("Sum of 2 and 7:", add(2, 7));
	117	```
	118
	119	> Bug: Return types are currently required. Omitting the `-> type` syntax currently makes the return type `void`, but this will be changed in the future to be an inferred return type.
	120
	121	The `return` expression can be used within functions to return early.
	122
	123	```lang:run
	124	fn something(a: int) -> bool {
	125	if a == 0 {
	126	return false;
	127	}
	128
	129	a > 100
	130	}
	131
	132	let print = @js("console").log;
	133	print(something(0xFF), something(0));
	134	```
	135
	136	## Number Literals
	137
	138	TODO
	139
	140	## String Literals
	141
	142	TODO
	143
	144	## JavaScript Interop
	145
	146	A global variable handle can be retrived with `@js`. We've seen this in the hello world. The type of this value is a special type called `opaque`, which disables type-checking.
	147
	148	> Note: A future update will improve on the type-safety of `opaque`
	149
	150	Many JS-exclusive actions are exposed via built-in functions, such as using `@jsNew` for calling constructors.
	151
	152	```lang:run
	153	let print = @js("console").log;
	154
	155	fn currentYear() -> int {
	156	@jsNew(@js("Date")).getFullYear()
	157	}
	158
	159	print(currentYear());
	160	```

doc/language-reference.md deleted-157

...	...	@@ -1,157 +0,0 @@
1		# New Language Project
2
3		This is a high-level, general purpose programming language.
4
5		Key goals of this project are:
6
7		- Ease of Use: Use of type inference keeps code concise and easy to read and write.
8		- Fast Iteration: The compiler is designed for quick incremental compilation and hot module reloading.
9		- TODO: some more
10
11		It primarily focuses on merging some great points of Rust, Zig, and JavaScript's language designs:
12
13		- Rust - Expressive control flow that is easy to write.
14		- Zig - Execute any code at compile time, everything is an expression.
15		- JavaScript - Garbage collected values with a versitile type system.
16
17		## Standard Library
18
19		Currently, there is no standard library. One will be introduced later.
20
21		## Hello World
22
23		```lang:run
24		let print = @js("console").log;
25		print("Hello, world!");
26		```
27
28		The `print` function is implemented by getting an external reference to the `console` JavaScript global, and then retriving the `log` property from it.
29
30		## Primative Data Types
31
32		\| name \| ts equivilent \| description \|
33		\| ---------- \| --------------------- \| ---------------------------------------------- \|
34		\| `int` \| `number` \| a number in signed 32-bit range \|
35		\| `float` \| `number` \| a 64-bit floating point number \|
36		\| `bool` \| `boolean` \| either `true` or `false` \|
37		\| `string` \| `string` \| a sequence of text characters \|
38		\| `void` \| `undefined` or `null` \| detectable lack of a value \|
39		\| `noreturn` \| `never` \| control flow prevents this value from existing \|
40
41		> note: `bigint` is planned but not added
42
43		## Built-in Constants
44
45		In addition to the above types being built-in constants, the following are available.
46
47		\| name \| description \|
48		\| ------- \| ------------------------------------- \|
49		\| `true` \| boolean true (result of `1 == 1`) \|
50		\| `false` \| boolean false (result of `1 == 2`) \|
51		\| `std` \| handle to the standard library module \|
52
53		These constants are reserved names, and it is an error to re-declare them.
54
55		```lang:build
56		let true = 1 == 0;
57		```
58
59		## Variables
60
61		There are two types of variables, `let` and `var`.
62
63		`let` bindings cannot be overridden, and should be thought of as aliases of the expression within. Referring to these identifiers is equivilent to inlining it's where it was used, aside from side effect duplication.
64
65		```lang:run
66		let hello: int = 1;
67		let world = hello + 2;
68		let hello = hello * world;
69
70		@js("console").log(hello);
71		```
72
73		`var` bindings create mutable memory locations at runtime. This means they can be re-assigned to:
74
75		```lang:run
76		var message = "first message";
77
78		if @js("Math").random() > 0.5 {
79		message = "second message";
80		}
81
82		@js("console").log(message);
83		```
84
85		Using `let` is preferred whenever possible, as it will lead to more readable code, and the compiler is able to optimize it better. The above can be rewritten as:
86
87		```lang:run
88		let message = if @js("Math").random() > 0.5 {
89		"second message"
90		} else {
91		"first message"
92		};
93
94		@js("console").log(message);
95		```
96
97		It should be noted that numeric literals are a comptime-only data type, meaning it cannot be assigned to a `var` without an explicit type annotation. Since `int` is a strict subset of valid `float`s, the coersion will implicitly be a float, but a compiler warning will be emitted.
98
99		```lang
100		var number = 0; // `number` is a `float`, but not explicit
101		var number: int = 0; // `number` is `int`
102		var number: float = 0; // `number` is `float`
103		```
104
105		## Functions
106
107		Functions are declared with the `fn` keyword. Parameter types must be annotated. The function body is a block, which may use implicit return to return.
108
109		```lang:run
110		fn add(a: int, b: int) -> int {
111		a + b
112		}
113
114		let print = @js("console").log;
115		print("Sum of 2 and 7" + add(2, 7));
116		```
117
118		> Bug: Return types are currently required. Omitting the `-> type` syntax currently makes the return type `void`, but this will be changed in the future to be an inferred return type.
119
120		The `return` expression can be used within functions to return early.
121
122		```lang:run
123		fn something(a: int) -> bool {
124		if a == 0 {
125		return false;
126		}
127
128		a > 100
129		}
130
131		let print = @js("console").log;
132		print(something(0xFF), something(0));
133		```
134
135		## Number Literals
136
137		TODO
138
139		## String Literals
140
141		TODO
142
143		## JavaScript Interop
144
145		A global variable handle can be retrived with `@js`. We've seen this in the hello world. The plan is to eventually replace this construct with something else, but implementing a function like this is extremely trivial for the compiler.
146
147		Many JS-exclusive actions are exposed via built-in functions, such as using `@jsNew` for calling constructors.
148
149		```lang:run
150		let print = @js("console").log;
151
152		fn currentYear() -> int {
153		@jsNew(@js("Date")).getFullYear()
154		}
155
156		print(currentYear());
157		```

src/core/InternPool.zig+8

...	...	@@ -288,6 +288,14 @@ pub const Value = enum(u32) {
288	288	pub fn toFloat(val: Value, ip: *const InternPool) ?f64 {
289	289	return switch (val.valueTag(ip)) {
290	290	inline .comptime_float, .float => \|tag\| val.valueData(ip, tag).get(ip).decode(),
	291	.int32 => @floatFromInt(val.valueData(ip, .int32)),
	292	else => null,
	293	};
	294	}
	295
	296	pub fn toInt32(val: Value, ip: *const InternPool) ?i32 {
	297	return switch (val.valueTag(ip)) {
	298	.int32 => val.valueData(ip, .int32),
291	299	else => null,
292	300	};
293	301	}

src/core/JsLink.zig+1-1

...	...	@@ -193,10 +193,10 @@ fn printBlockContents(
193	193	for (list.slice(tree)) \|stmt\| {
194	194	try link.printExpr(tree, stmt, writer);
195	195	switch (tree.nodeTag(stmt)) {
196		else => try writer.writeAll(";"),
197	196	.stmt_block,
198	197	.stmt_if,
199	198	=> {},
	199	else => try writer.writeAll(";"),
200	200	}
201	201	try writer.writeAll("\n");
202	202	}

src/core/Parse.zig+30-7

...	...	@@ -245,8 +245,20 @@ fn parseModuleMembers(p: *Parse) Error!SubList {
245	245	return p.failUnexpectedToken(.statement);
246	246	}
247	247
	248	const first_tag = p.currentTag();
	249
248	250	try p.scratch.append(p.gpa, try p.parseExpression(true));
249		_ = try p.expect(.semi);
	251
	252	switch (first_tag) {
	253	// some control flow and blocks are allowed to not have a
	254	// trailing semicolons. this is a stylistic choice.
	255	.kw_if,
	256	.start_brace,
	257	=> {},
	258
	259	// TODO: make semicolons optional, but emit a warning
	260	else => _ = try p.expect(.semi),
	261	}
250	262	},
251	263	};
252	264
...	...	@@ -423,9 +435,7 @@ fn parseBlockContents(p: *Parse) Error!struct { Node.Tag, SubList } {
423	435	if (p.currentTag() == .end_brace) break .block_implicit_return;
424	436	},
425	437
426		// TODO: make kw_break and kw_return REQUIRE a semicolon, as
427		// they are noreturn statements, doing implicit break on
428		// them makes no sense
	438	// TODO: make semicolons optional, but emit a warning
429	439
430	440	else => {
431	441	if (p.eatToken(.semi) == null) {
...	...	@@ -456,13 +466,13 @@ fn parseVarDecl(p: *Parse) Error!NodeIndex {
456	466	_ = try p.expect(.id);
457	467
458	468	const data_type: OptionalNodeIndex = if (p.eatToken(.colon)) \|_\|
459		optionalNodeIndex(try p.parseExpression(false))
	469	optionalNodeIndex(try p.tryParseExpression(false))
460	470	else
461	471	.none;
462	472
463	473	_ = try p.expect(.equal);
464	474
465		const data_init = try p.parseExpression(true);
	475	const data_init = try p.tryParseExpression(true);
466	476
467	477	_ = try p.expect(.semi);
468	478
...	...	@@ -711,6 +721,13 @@ fn parseExpression(p: *Parse, allow_assign_operators: bool) Error!NodeIndex {
711	721	return expr;
712	722	}
713	723
	724	fn tryParseExpression(p: *Parse, allow_assign_operators: bool) Error!NodeIndex {
	725	return if (p.isExpressionStart())
	726	p.parseExpression(allow_assign_operators)
	727	else
	728	p.failUnexpectedToken(.expression);
	729	}
	730
714	731	fn parseExpressionPart(p: *Parse) Error!NodeIndex {
715	732	const trace = t.trace(@src());
716	733	defer trace.end();
...	...	@@ -1040,8 +1057,14 @@ fn fail(p: *Parse, ast_error: AstError) Error {
1040	1057	}
1041	1058
1042	1059	fn failUnexpectedToken(p: *Parse, kind: ExpectedContext) Error {
	1060	var got = p.fullToken(p.pos);
	1061	if (got.tag == .eof) {
	1062	// note: it is impossible to get a failure at an empty file, so subtraction is fine
	1063	const prev = p.fullToken(p.pos - 1);
	1064	got.offset = prev.offset + prev.len(p.source);
	1065	}
1043	1066	return p.fail(.{ .unexpected_context = .{
1044		.got = p.fullToken(p.pos),
	1067	.got = got,
1045	1068	.expected = kind,
1046	1069	} });
1047	1070	}

src/core/Report.zig+25-6

...	...	@@ -247,10 +247,24 @@ pub const Err = union(enum) {
247	247	node: NodeIndex,
248	248
249	249	fn build(data: @This(), ctx: *MsgContext) !void {
	250	try ctx.msgNode(data.node, "invalid assignment target", .{});
	251	}
	252	},
	253	bin_op_mismatch: struct {
	254	src: LazyNode,
	255	lhs: Type,
	256	rhs: Type,
	257
	258	fn build(data: @This(), ctx: *MsgContext) !void {
	259	const node = try data.src.get(ctx);
250	260	try ctx.msgNode(
251		data.node,
252		"invalid assignment target",
253		.{},
	261	node,
	262	"\"{s}\" operator is not compatible with types \"{}\" and \"{}\"",
	263	.{
	264	ctx.ast.tokenSlice(ctx.ast.nodeToken(node)),
	265	data.lhs.displayName(&ctx.comp.intern_pool),
	266	data.rhs.displayName(&ctx.comp.intern_pool),
	267	},
254	268	);
255	269	}
256	270	},
...	...	@@ -458,7 +472,13 @@ pub fn formatItem(
458	472	const file = try comp.getFile(item.url.slice(report));
459	473
460	474	const need_load_ast = !(item_type == .@"error" and item.data.@"error" == .ast);
461		const ast_data = if (need_load_ast) try file.getAst(comp) else undefined;
	475	const ast_data = if (need_load_ast)
	476	try file.getAst(comp)
	477	else brk: {
	478	var ast: Ast = undefined;
	479	ast.utf8_source = (try file.getSource(comp.gpa)).code;
	480	break :brk ast;
	481	};
462	482
463	483	var ctx: MsgContext = .{
464	484	.arena = alloc,
...	...	@@ -700,13 +720,12 @@ pub const Highlight = union(enum) {
700	720	.byte = token.offset,
701	721	};
702	722	},
703
704	723	.token_stored => \|token\| loc: {
705	724	const location = Tokenizer.tokenLocation(ast.utf8_source, 0, token.offset);
706	725
707	726	break :loc .{
708	727	.loc = location,
709		.slice = token.slice(ast.utf8_source),
	728	.slice = if (token.tag != .eof) token.slice(ast.utf8_source) else "\n",
710	729	.byte = token.offset,
711	730	};
712	731	},

src/core/Sema.zig+139-41

...	...	@@ -315,28 +315,30 @@ fn analyzeCodeInner(s: *Sema, in: Teg.Inst.Index) Error!Air.Inst.Ref {
315	315	.eq,
316	316	=> \|tag\| {
317	317	const bin: Teg.Inst.Bin = s.code.instPayload(tag, in);
	318	const op = @field(BinaryOp, @tagName(tag));
318	319
319	320	const lhs = try s.resolveRefRuntime(bin.lhs);
320	321	const rhs = try s.resolveRefRuntime(bin.rhs);
321	322
322		if (lhs.kind == .value and rhs.kind == .value) {
323		return (try s.comptimeBin(
324		@field(BinaryOp, @tagName(tag)),
325		lhs.decode().value,
326		rhs.decode().value,
327		));
328		}
	323	const left_ty = s.typeOfRef(lhs);
	324	const right_ty = s.typeOfRef(rhs);
329	325
330		const left_type = s.typeOfRef(lhs);
331		_ = left_type; // autofix
332		const right_type = s.typeOfRef(rhs);
333		_ = right_type; // autofix
	326	const ty = try s.binTypeResolution(in, op, left_ty, right_ty);
334	327
335		// TODO: validate type for operation
	328	const left = try s.coerceRef(in.op(.lhs), lhs, ty);
	329	const right = try s.coerceRef(in.op(.rhs), lhs, ty);
	330
	331	if (left.kind == .value and right.kind == .value) {
	332	return (try s.comptimeBin(
	333	op,
	334	left.decode().value,
	335	right.decode().value,
	336	)).toAir();
	337	}
336	338
337	339	return s.addInstRef(tag.airTag(), .{
338		.lhs = lhs,
339		.rhs = rhs,
	340	.lhs = left,
	341	.rhs = right,
340	342	});
341	343	},
342	344	inline .negate, .bool_not => \|tag\| {
...	...	@@ -881,35 +883,133 @@ const BinaryOp = enum {
881	883	eq,
882	884	};
883	885
884		fn comptimeBin(s: *Sema, op: BinaryOp, lhs: Value, rhs: Value) !Air.Inst.Ref {
885		switch (op) {
886		.add => {
887		const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck / string add");
888		const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck / string add");
889		return (try s.ip.get(s.gpa, .{
890		.float = l + r,
891		})).toAir();
	886	fn binTypeResolution(s: *Sema, src: Teg.Inst.Index, op: BinaryOp, lhs: Type, rhs: Type) !Type {
	887	if (lhs == .@"opaque") return rhs;
	888	if (rhs == .@"opaque") return rhs;
	889
	890	switch (lhs) {
	891	.string => if (op == .add) switch (rhs) {
	892	.string => return .string,
	893	else => {},
	894	},
	895	.numeric_literal => switch (rhs) {
	896	.int32,
	897	.float,
	898	.numeric_literal,
	899	=> return rhs,
	900	else => {},
	901	},
	902	.int32 => switch (rhs) {
	903	.int32 => return .int32,
	904	.numeric_literal => return .int32,
	905	else => {},
	906	},
	907	.float => switch (rhs) {
	908	.int32 => return .float,
	909	.numeric_literal => return .float,
	910	else => {},
892	911	},
893		inline .sub, .mul, .div => \|tag\| {
894		const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck");
895		const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck");
	912	else => {},
	913	}
896	914
897		const result = switch (tag) {
898		.sub => l - r,
899		.mul => l * r,
900		.div => l / r,
901		else => comptime unreachable,
902		};
	915	return s.fail(.bin_op_mismatch, .{
	916	.src = s.nodeLoc(src, .main_node),
	917	.lhs = lhs,
	918	.rhs = rhs,
	919	});
	920	}
	921
	922	fn comptimeBin(s: *Sema, op: BinaryOp, lhs: Value, rhs: Value) !Value {
	923	std.debug.print("{} {s} {}\n", .{
	924	lhs.dump(s.ip),
	925	@tagName(op),
	926	rhs.dump(s.ip),
	927	});
	928
	929	const left_ty = lhs.typeOf(s.ip);
	930	assert(rhs.typeOf(s.ip) == left_ty);
903	931
904		return (try s.ip.get(s.gpa, .{
905		.float = result,
906		})).toAir();
	932	switch (op) {
	933	.add => switch (left_ty) {
	934	.numeric_literal, .float => {
	935	const l = lhs.toFloat(s.ip) orelse
	936	unreachable; // must be coerced
	937	const r = rhs.toFloat(s.ip) orelse
	938	unreachable; // must be coerced
	939
	940	const result = l + r;
	941
	942	return try s.makeValue(if (left_ty == .numeric_literal)
	943	.{ .comptime_float = result }
	944	else
	945	.{ .float = result });
	946	},
	947	.int32 => {
	948	const l = lhs.toInt32(s.ip) orelse
	949	unreachable; // must be coerced
	950	const r = rhs.toInt32(s.ip) orelse
	951	unreachable; // must be coerced
	952
	953	const result = std.math.add(i32, l, r) catch
	954	@panic("TODO: out of range");
	955
	956	return try s.makeValue(.{ .int32 = result });
	957	},
	958	.string => {
	959	const l = lhs.asString(s.ip).toSlice(s.ip);
	960	const r = rhs.asString(s.ip).toSlice(s.ip);
	961	var sfa = std.heap.stackFallback(4096, s.arena);
	962	const alloc = sfa.get();
	963	const joined = try std.mem.concat(alloc, u8, &.{ l, r });
	964	defer alloc.free(joined);
	965	return try s.makeValue(.{ .string = joined });
	966	},
	967	else => unreachable,
	968	},
	969	inline .sub, .mul, .div => \|tag\| switch (left_ty) {
	970	.numeric_literal, .float => {
	971	const l = lhs.toFloat(s.ip) orelse
	972	unreachable; // must be coerced
	973	const r = rhs.toFloat(s.ip) orelse
	974	unreachable; // must be coerced
	975
	976	const result = switch (tag) {
	977	.sub => l - r,
	978	.mul => l * r,
	979	.div => l / r,
	980	else => comptime unreachable,
	981	};
	982
	983	return try s.makeValue(if (left_ty == .numeric_literal)
	984	.{ .comptime_float = result }
	985	else
	986	.{ .float = result });
	987	},
	988	.int32 => {
	989	const l = lhs.toInt32(s.ip) orelse
	990	unreachable; // must be coerced
	991	const r = rhs.toInt32(s.ip) orelse
	992	unreachable; // must be coerced
	993
	994	const result = switch (tag) {
	995	.sub => std.math.sub(i32, l, r),
	996	.mul => std.math.mul(i32, l, r),
	997	.div => return try s.makeValue(.{ .float = @as(f64, @floatFromInt(l)) / @as(f64, @floatFromInt(r)) }),
	998	else => comptime unreachable,
	999	} catch @panic("TODO: out of range");
	1000
	1001	return try s.makeValue(.{ .int32 = result });
	1002	},
	1003	else => unreachable,
907	1004	},
908	1005	inline .lt,
909	1006	.gt,
910	1007	=> \|tag\| {
911		const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck");
912		const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck");
	1008	const l = lhs.toFloat(s.ip) orelse
	1009	unreachable; // must be coerced
	1010	const r = rhs.toFloat(s.ip) orelse
	1011	unreachable; // must be coerced
	1012
913	1013	const result = switch (tag) {
914	1014	.lt => l < r,
915	1015	.gt => l > r,
...	...	@@ -918,17 +1018,15 @@ fn comptimeBin(s: *Sema, op: BinaryOp, lhs: Value, rhs: Value) !Air.Inst.Ref {
918	1018	return @as(
919	1019	InternPool.Value,
920	1020	if (result) .bool_true else .bool_false,
921		).toAir();
	1021	);
922	1022	},
923	1023	.eq,
924	1024	=> {
925		const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck / non-numeric eq");
926		const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck / non-numeric eq");
927		const result = l == r;
	1025	const result = lhs == rhs;
928	1026	return @as(
929	1027	InternPool.Value,
930	1028	if (result) .bool_true else .bool_false,
931		).toAir();
	1029	);
932	1030	},
933	1031	}
934	1032	}

src/core/Tokenizer.zig+2-1

...	...	@@ -152,7 +152,6 @@ pub const Token = struct {
152	152	.invalid => unreachable,
153	153
154	154	// do not have a static lexeme
155		.eof,
156	155	.id,
157	156	.builtin_id,
158	157	.number_int,
...	...	@@ -163,6 +162,8 @@ pub const Token = struct {
163	162	.module_doc_comment,
164	163	=> null,
165	164
	165	.eof => unreachable, // always must be special cased
	166
166	167	.@"and" => "&",
167	168	.and_and => "&&",
168	169	.bang => "!",

src/langref/generate-langref.zig deleted-364

...	...	@@ -1,364 +0,0 @@
1		//! Language reference compiler. Converts markdown into HTML, using
2		//! the Tokenizer to highlight code, and the compiler and node.js to
3		//! show code results. Only ever built in Debug mode. We are not
4		//! smart about memory. Written at 3am on a Thursday. Enjoy
5		pub const std = @import("std");
6		pub const core = @import("core");
7
8		const ArrayList = std.ArrayList;
9
10		pub fn main() !void {
11		comptime std.debug.assert(@import("builtin").mode == .Debug);
12		var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
13		const gpa = arena.allocator();
14
15		var args = try std.process.argsWithAllocator(gpa);
16		_ = args.next().?;
17
18		const compiler_path = args.next().?;
19		const markdown_src = try std.fs.cwd().readFileAlloc(gpa, args.next().?, std.math.maxInt(usize));
20		const template_src = try std.fs.cwd().readFileAlloc(gpa, args.next().?, std.math.maxInt(usize));
21		const out_path = args.next().?;
22
23		var output = std.ArrayList(u8).init(gpa);
24		defer output.deinit();
25		try processMarkdown(
26		gpa,
27		markdown_src,
28		compiler_path,
29		output.writer(),
30		);
31
32		const applied_template = try std.mem.replaceOwned(
33		u8,
34		gpa,
35		template_src,
36		"{contents}",
37		output.items,
38		);
39		defer gpa.free(applied_template);
40
41		const out_file = try std.fs.createFileAbsolute(out_path, .{});
42		defer out_file.close();
43		try out_file.writeAll(applied_template);
44		}
45
46		const BlockType = enum {
47		para,
48		h1,
49		h2,
50		h3,
51		code_run,
52		code_build,
53		code,
54		code_generic,
55		ul,
56		table,
57		blockquote,
58		};
59
60		const InlineType = enum {
61		bold,
62		code,
63		};
64
65		const MarkdownTokenizer = struct {
66		/// Remaining text
67		str: []const u8,
68
69		fn skipWhitespace(t: *MarkdownTokenizer) void {
70		t.str = std.mem.trimLeft(u8, t.str, " \r\n");
71		}
72
73		fn advance(t: *MarkdownTokenizer, n: usize) []const u8 {
74		defer {
75		t.str = t.str[n..];
76		}
77		return t.str[0..n];
78		}
79
80		fn advanceLine(t: *MarkdownTokenizer) []const u8 {
81		return std.mem.trim(u8, t.advanceLineRaw(), " \r\n");
82		}
83
84		fn advanceLineRaw(t: *MarkdownTokenizer) []const u8 {
85		const n = std.mem.indexOfScalar(u8, t.str, '\n');
86		const line = t.advance(if (n) \|x\| x + 1 else t.str.len);
87		return line;
88		}
89
90		fn blockType(t: *MarkdownTokenizer) BlockType {
91		switch (t.str[0]) {
92		'#' => {
93		defer {
94		while (t.str[0] == '#')
95		_ = t.advance(1);
96		}
97
98		if (t.str[1] != '#') return .h1;
99		if (t.str[2] != '#') return .h2;
100		if (t.str[3] != '#') return .h3;
101		@panic("headers nested too deep");
102		},
103		'>' => {
104		_ = t.advance(1);
105		return .blockquote;
106		},
107		'\|' => {
108		return .table;
109		},
110		'`' => {
111		if (std.mem.startsWith(u8, t.str, "```")) {
112		defer _ = t.advanceLine();
113		if (std.mem.startsWith(u8, t.str, "```lang:run"))
114		return .code_run;
115		if (std.mem.startsWith(u8, t.str, "```lang:build"))
116		return .code_build;
117		if (std.mem.startsWith(u8, t.str, "```lang"))
118		return .code;
119		return .code_generic;
120		}
121		return .para;
122		},
123		'-' => {
124		return .ul;
125		},
126		else => {
127		return .para;
128		},
129		}
130		}
131		};
132
133		fn processMarkdown(gpa: std.mem.Allocator, src: []const u8, compiler_path: []const u8, writer: anytype) !void {
134		var t = MarkdownTokenizer{ .str = src };
135
136		t.skipWhitespace();
137		while (t.str.len > 0) : (t.skipWhitespace()) {
138		switch (t.blockType()) {
139		.para,
140		=> {
141		try writer.writeAll("<p>\n");
142
143		var para = ArrayList(u8).init(gpa);
144		try para.appendSlice(t.advanceLine());
145
146		var next_line = t.advanceLine();
147		while (t.str.len > 0 and next_line.len > 0) {
148		try para.append(' ');
149		try para.appendSlice(next_line);
150		next_line = t.advanceLine();
151		}
152
153		try processInlineMarkdown(gpa, para.items, writer);
154
155		try writer.writeAll("\n</p>\n");
156		},
157		.h1,
158		.h2,
159		.h3,
160		=> \|tag\| {
161		const html_tag = @tagName(tag);
162		const name = t.advanceLine();
163		try writer.print("<{0s}>{1s}</{0s}>\n", .{
164		html_tag,
165		name,
166		});
167		},
168		.code_run,
169		.code_build,
170		.code,
171		.code_generic,
172		=> \|tag\| {
173		var code = ArrayList(u8).init(gpa);
174
175		var next_line = t.advanceLineRaw();
176		while (t.str.len > 0 and !std.mem.startsWith(u8, next_line, "```")) {
177		try code.appendSlice(next_line);
178		next_line = t.advanceLineRaw();
179		}
180
181		try writer.writeAll("<pre><code>");
182		try writeTokenizedCode(gpa, code.items, writer);
183		try writer.writeAll("</code></pre>");
184
185		try code.append('\n');
186
187		switch (tag) {
188		.code_run,
189		.code_build,
190		=> {
191		var build = std.process.Child.init(&.{ compiler_path, "/dev/stdin" }, gpa);
192		build.stdin_behavior = .Pipe;
193		build.stdout_behavior = .Pipe;
194		build.stderr_behavior = .Pipe;
195		var env_map = try std.process.getEnvMap(gpa);
196		defer env_map.deinit();
197		try env_map.put("YES_COLOR", "1");
198		build.env_map = &env_map;
199		try build.spawn();
200		try build.stdin.?.writeAll(code.items);
201		var out = ArrayList(u8).init(gpa);
202		var err = ArrayList(u8).init(gpa);
203		try build.collectOutput(&out, &err, std.math.maxInt(usize));
204		const result = try build.wait();
205		_ = result;
206		if (std.mem.trim(u8, err.items, " \n\r").len > 0) {
207		try writer.writeAll("<pre class='term'><code>");
208		try writeAnsi(gpa, err.items, writer);
209		try writer.writeAll("</code></pre>");
210		}
211		if (std.mem.trim(u8, out.items, " \n\r").len > 0) {
212		try writer.writeAll("<pre><code>");
213		try writer.writeAll("// Compiled JavaScript Code");
214		try writer.writeAll(out.items[std.mem.indexOfScalar(u8, out.items, '\n').?..]);
215		try writer.writeAll("</code></pre>");
216		}
217		},
218		else => {},
219		}
220		},
221		.ul,
222		=> {
223		try writer.writeAll("<ul>");
224		while (t.str.len > 0 and (t.str[0] == '-')) {
225		_ = t.advance(1);
226		const line = t.advanceLine();
227		try writer.writeAll("<li>");
228		try processInlineMarkdown(gpa, line, writer);
229		try writer.writeAll("</li>");
230		}
231		try writer.writeAll("</ul>");
232		},
233		.table,
234		=> {
235		_ = t.advanceLine();
236		},
237		.blockquote,
238		=> {
239		try writer.writeAll("<blockquote>\n");
240
241		var para = ArrayList(u8).init(gpa);
242		try para.appendSlice(t.advanceLine());
243
244		var next_line = t.advanceLine();
245		while (t.str.len > 0 and next_line.len > 0) {
246		try para.append(' ');
247		try para.appendSlice(next_line);
248		next_line = t.advanceLine();
249		}
250
251		try processInlineMarkdown(gpa, para.items, writer);
252
253		try writer.writeAll("\n</blockquote>\n");
254		},
255		}
256		}
257		}
258
259		pub fn nextInlineToken(src: []const u8) ?struct { InlineType, usize } {
260		var pos: usize = 0;
261		while (true) : (pos += 1) {
262		pos = std.mem.indexOfAnyPos(u8, src, pos, "*`") orelse return null;
263
264		switch (src[pos]) {
265		'*' => switch (src[pos + 1]) {
266		'*' => return .{ .bold, pos },
267		else => continue,
268		},
269		'`' => return .{ .code, pos },
270		else => unreachable,
271		}
272		}
273		}
274
275		fn processInlineMarkdown(gpa: std.mem.Allocator, src: []const u8, writer: anytype) !void {
276		var bold = false;
277		var remain = src;
278
279		while (nextInlineToken(remain)) \|tok\| {
280		try writer.writeAll(remain[0..tok[1]]);
281		remain = remain[tok[1]..];
282		switch (tok[0]) {
283		.bold => {
284		bold = !bold;
285		try writer.writeAll(switch (bold) {
286		true => "<strong>",
287		false => "</strong>",
288		});
289		remain = remain[2..];
290		},
291		.code => {
292		const end = std.mem.indexOfScalarPos(u8, remain, 1, '`') orelse @panic("invalid code block");
293		const code = remain[1..end];
294		try writer.writeAll("<code>");
295		try writeTokenizedCode(gpa, code, writer);
296		try writer.writeAll("</code>");
297		remain = remain[end + 1 ..];
298		},
299		}
300		}
301		try writer.writeAll(remain);
302
303		if (bold) {
304		std.debug.panic("Unterminated Bold in '{s}'", .{src});
305		}
306		}
307
308		// TODO: Change this to use Tree-sitter, which would also make it very simple
309		// to highlight JavaScript code in addition to the language.
310		fn writeTokenizedCode(gpa: std.mem.Allocator, code: []const u8, writer: anytype) !void {
311		const code_z = try gpa.dupeZ(u8, code);
312		defer gpa.free(code_z);
313
314		var t = core.Tokenizer{ .code = code_z };
315
316		t.skipWhitespace();
317		var last: usize = 0;
318		while (true) {
319		t.skipWhitespace();
320		const tok = t.next() catch {
321		t.pos += 1;
322		continue;
323		} orelse break;
324		try writer.writeAll(t.code[last..tok.offset]);
325		try writer.print("<span class='tok-{s}{s}'>{s}</span>", .{
326		@tagName(tok.tag),
327		if (tok.tag.isKeyword()) " kw" else "",
328		t.code[tok.offset..t.pos],
329		});
330		last = t.pos;
331		}
332		}
333
334		fn writeAnsi(gpa: std.mem.Allocator, code: []const u8, writer: anytype) !void {
335		var remain = code;
336		var count: usize = 0;
337		while (std.mem.indexOfScalar(u8, remain, '\x1b')) \|esc_i\| {
338		try writer.writeAll(remain[0..esc_i]);
339		switch (remain[esc_i + 1]) {
340		'[' => switch (remain[esc_i + 2]) {
341		'0'...'9' => {
342		const end = std.mem.indexOfScalarPos(u8, remain, esc_i + 2, 'm').?;
343		const int = std.fmt.parseInt(u8, remain[esc_i + 2 .. end], 10) catch unreachable;
344		switch (int) {
345		else => {
346		try writer.print("<span class='ansi-{d}'>", .{int});
347		count += 1;
348		},
349		0 => {
350		for (0..count) \|_\| {
351		try writer.writeAll("</span>");
352		}
353		count = 0;
354		},
355		}
356		remain = remain[end + 1 ..];
357		},
358		else => \|x\| std.debug.panic("Unknown esc + 2: {d}", .{x}),
359		},
360		else => \|x\| std.debug.panic("Unknown esc + 1: {d}", .{x}),
361		}
362		}
363		_ = gpa;
364		}

src/langref/template.html deleted-100

...	...	@@ -1,100 +0,0 @@
1		<!doctype html>
2		<html lang="en">
3		<head>
4		<meta charset="UTF-8">
5		<meta name="viewport" content="width=device-width, initial-scale=1.0">
6		<title>Language Reference</title>
7		<style>
8		* { margin: 0; padding: 0; box-sizing: border-box; }
9		body {
10		background-color: #fefefe;
11		font-family: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu, Cantarell, "Open Sans", "Helvetica Neue", sans-serif;
12		max-width: 45rem;
13		padding: 1rem;
14		margin: auto;
15		}
16		h1,h2,h3,h4,h5,h6,p,ul,pre,blockquote {
17		margin-bottom: 1rem;
18		}
19		ul {
20		padding-left: 2rem;
21		}
22		li {
23		margin-bottom: 0.25rem;
24		}
25		blockquote {
26		background-color: #ddd;
27		border-left: 4px solid black;
28		padding-left: 16px;
29		padding-top: 8px;
30		padding-bottom: 8px;
31		}
32		pre > code {
33		border: none;
34		padding: 0;
35		background-color: transparent;
36		}
37		code, pre {
38		font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
39		font-size: 0.9rem;
40		border: 1px solid #9ca0b0;
41		background-color: #eff1f5;
42		border-radius: 4px;
43		padding: 2px 4px;
44		}
45		pre {
46		padding: 0.75rem 0.75rem;
47		}
48		.term {
49		border: 1px solid #11111b;
50		background-color: #24273a;
51		color: #cdd6f4;
52		}
53
54		.ansi-1 {
55		font-weight: bold;
56		}
57		.ansi-31 {
58		color: #ed8796;
59		}
60		.ansi-32 {
61		color: #a6e3a1;
62		}
63		.ansi-33 {
64		color: #eed49f;
65		}
66		.ansi-34 {
67		color: #8aadf4;
68		}
69		.ansi-35 {
70		color: #f5c2e7;
71		}
72		.ansi-36 {
73		color: #94e2d5;
74		}
75		.ansi-2 {
76		color: #8087a2;
77		}
78
79		.kw {
80		color: #8839ef;
81		font-weight: bold;
82		}
83		.tok-string {
84		color: #40a02b;
85		}
86		.tok-builtin_id {
87		color: #d20f39;
88		font-weight: bold;
89		}
90		.tok-number_int, .tok-number_float {
91		color: #fe640b;
92		}
93		</style>
94		</head>
95		<body>
96		<main>
97		{contents}
98		</main>
99		</body>
100		</html>

src/manual/generate-man.zig created+387

...	...	@@ -0,0 +1,387 @@
	1	//! Language reference compiler. Converts markdown into HTML, using
	2	//! the Tokenizer to highlight code, and the compiler and node.js to
	3	//! show code results. Only ever built in Debug mode. We are not
	4	//! smart about memory. Written at 3am on a Thursday. Enjoy
	5	pub const std = @import("std");
	6	pub const core = @import("core");
	7
	8	const ArrayList = std.ArrayList;
	9
	10	pub fn main() !void {
	11	comptime std.debug.assert(@import("builtin").mode == .Debug);
	12	var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
	13	const gpa = arena.allocator();
	14
	15	var args = try std.process.argsWithAllocator(gpa);
	16	_ = args.next().?;
	17
	18	const compiler_path = args.next().?;
	19	const markdown_src = try std.fs.cwd().readFileAlloc(gpa, args.next().?, std.math.maxInt(usize));
	20	const template_src = try std.fs.cwd().readFileAlloc(gpa, args.next().?, std.math.maxInt(usize));
	21	const out_path = args.next().?;
	22
	23	var output = std.ArrayList(u8).init(gpa);
	24	defer output.deinit();
	25	try processMarkdown(
	26	gpa,
	27	markdown_src,
	28	compiler_path,
	29	output.writer(),
	30	);
	31
	32	const applied_template = try std.mem.replaceOwned(
	33	u8,
	34	gpa,
	35	template_src,
	36	"{contents}",
	37	output.items,
	38	);
	39	defer gpa.free(applied_template);
	40
	41	const out_file = try std.fs.createFileAbsolute(out_path, .{});
	42	defer out_file.close();
	43	try out_file.writeAll(applied_template);
	44	}
	45
	46	const BlockType = enum {
	47	para,
	48	h1,
	49	h2,
	50	h3,
	51	code_run,
	52	code_build,
	53	code,
	54	code_generic,
	55	ul,
	56	table,
	57	blockquote,
	58	};
	59
	60	const InlineType = enum {
	61	bold,
	62	code,
	63	};
	64
	65	const MarkdownTokenizer = struct {
	66	/// Remaining text
	67	str: []const u8,
	68
	69	fn skipWhitespace(t: *MarkdownTokenizer) void {
	70	t.str = std.mem.trimLeft(u8, t.str, " \r\n");
	71	}
	72
	73	fn advance(t: *MarkdownTokenizer, n: usize) []const u8 {
	74	defer {
	75	t.str = t.str[n..];
	76	}
	77	return t.str[0..n];
	78	}
	79
	80	fn advanceLine(t: *MarkdownTokenizer) []const u8 {
	81	return std.mem.trim(u8, t.advanceLineRaw(), " \r\n");
	82	}
	83
	84	fn advanceLineRaw(t: *MarkdownTokenizer) []const u8 {
	85	const n = std.mem.indexOfScalar(u8, t.str, '\n');
	86	const line = t.advance(if (n) \|x\| x + 1 else t.str.len);
	87	return line;
	88	}
	89
	90	fn blockType(t: *MarkdownTokenizer) BlockType {
	91	switch (t.str[0]) {
	92	'#' => {
	93	defer {
	94	while (t.str[0] == '#')
	95	_ = t.advance(1);
	96	}
	97
	98	if (t.str[1] != '#') return .h1;
	99	if (t.str[2] != '#') return .h2;
	100	if (t.str[3] != '#') return .h3;
	101	@panic("headers nested too deep");
	102	},
	103	'>' => {
	104	_ = t.advance(1);
	105	return .blockquote;
	106	},
	107	'\|' => {
	108	return .table;
	109	},
	110	'`' => {
	111	if (std.mem.startsWith(u8, t.str, "```")) {
	112	defer _ = t.advanceLine();
	113	if (std.mem.startsWith(u8, t.str, "```lang:run"))
	114	return .code_run;
	115	if (std.mem.startsWith(u8, t.str, "```lang:build"))
	116	return .code_build;
	117	if (std.mem.startsWith(u8, t.str, "```lang"))
	118	return .code;
	119	return .code_generic;
	120	}
	121	return .para;
	122	},
	123	'-' => {
	124	return .ul;
	125	},
	126	else => {
	127	return .para;
	128	},
	129	}
	130	}
	131	};
	132
	133	fn processMarkdown(gpa: std.mem.Allocator, src: []const u8, compiler_path: []const u8, writer: anytype) !void {
	134	var t = MarkdownTokenizer{ .str = src };
	135
	136	t.skipWhitespace();
	137	while (t.str.len > 0) : (t.skipWhitespace()) {
	138	switch (t.blockType()) {
	139	.para,
	140	=> {
	141	try writer.writeAll("<p>\n");
	142
	143	var para = ArrayList(u8).init(gpa);
	144	try para.appendSlice(t.advanceLine());
	145
	146	var next_line = t.advanceLine();
	147	while (t.str.len > 0 and next_line.len > 0) {
	148	try para.append(' ');
	149	try para.appendSlice(next_line);
	150	next_line = t.advanceLine();
	151	}
	152
	153	try processInlineMarkdown(gpa, para.items, writer);
	154
	155	try writer.writeAll("\n</p>\n");
	156	},
	157	.h1,
	158	.h2,
	159	.h3,
	160	=> \|tag\| {
	161	const html_tag = @tagName(tag);
	162	const name = t.advanceLine();
	163	try writer.print("<{0s}>{1s}</{0s}>\n", .{
	164	html_tag,
	165	name,
	166	});
	167	},
	168	.code_run,
	169	.code_build,
	170	.code,
	171	.code_generic,
	172	=> \|tag\| {
	173	var code = ArrayList(u8).init(gpa);
	174
	175	var next_line = t.advanceLineRaw();
	176	while (t.str.len > 0 and !std.mem.startsWith(u8, next_line, "```")) {
	177	try code.appendSlice(next_line);
	178	next_line = t.advanceLineRaw();
	179	}
	180
	181	try writer.writeAll("<pre><code>");
	182	try writeTokenizedCode(gpa, code.items, writer);
	183	try writer.writeAll("</code></pre>");
	184
	185	try code.append('\n');
	186
	187	switch (tag) {
	188	.code_run,
	189	.code_build,
	190	=> {
	191	var build = std.process.Child.init(&.{ compiler_path, "/dev/stdin" }, gpa);
	192	build.stdin_behavior = .Pipe;
	193	build.stdout_behavior = .Pipe;
	194	build.stderr_behavior = .Pipe;
	195	var env_map = try std.process.getEnvMap(gpa);
	196	defer env_map.deinit();
	197	try env_map.put("YES_COLOR", "1");
	198	build.env_map = &env_map;
	199	try build.spawn();
	200	try build.stdin.?.writeAll(code.items);
	201	var out = ArrayList(u8).init(gpa);
	202	var err = ArrayList(u8).init(gpa);
	203	try build.collectOutput(&out, &err, std.math.maxInt(usize));
	204	const result = try build.wait();
	205	_ = result;
	206	if (std.mem.trim(u8, err.items, " \n\r").len > 0) {
	207	try writer.writeAll("<pre class='term'><code>");
	208	try writeAnsi(gpa, err.items, writer);
	209	try writer.writeAll("</code></pre>");
	210	}
	211	if (std.mem.trim(u8, out.items, " \n\r").len > 0) {
	212	try writer.writeAll("<pre><code>");
	213	try writer.writeAll("// Compiled JavaScript Code");
	214	try writer.writeAll(out.items[std.mem.indexOfScalar(u8, out.items, '\n').?..]);
	215	try writer.writeAll("</code></pre>");
	216	}
	217	},
	218	else => {},
	219	}
	220	},
	221	.ul,
	222	=> {
	223	try writer.writeAll("<ul>");
	224	while (t.str.len > 0 and (t.str[0] == '-')) {
	225	_ = t.advance(1);
	226	const line = t.advanceLine();
	227	try writer.writeAll("<li>");
	228	try processInlineMarkdown(gpa, line, writer);
	229	try writer.writeAll("</li>");
	230	}
	231	try writer.writeAll("</ul>");
	232	},
	233	.table,
	234	=> {
	235	try writer.writeAll("<table><tbody>");
	236	{
	237	try writer.writeAll("<tr>");
	238	var it = std.mem.tokenizeScalar(u8, t.advanceLine()[1..], '\|');
	239	while (it.next()) \|tok\| {
	240	try writer.writeAll("<th>");
	241	try processInlineMarkdown(gpa, tok, writer);
	242	try writer.writeAll("</th>");
	243	}
	244	try writer.writeAll("</tr>");
	245	}
	246	_ = t.advanceLine();
	247	t.skipWhitespace();
	248	while (t.str.len > 0 and t.str[0] == '\|') {
	249	try writer.writeAll("<tr>");
	250	var it = std.mem.tokenizeScalar(u8, t.advanceLine()[1..], '\|');
	251	while (it.next()) \|tok\| {
	252	try writer.writeAll("<td>");
	253	try processInlineMarkdown(gpa, tok, writer);
	254	try writer.writeAll("</td>");
	255	}
	256	try writer.writeAll("</tr>");
	257	}
	258	try writer.writeAll("</tbody></table>");
	259	},
	260	.blockquote,
	261	=> {
	262	try writer.writeAll("<blockquote>\n");
	263
	264	var para = ArrayList(u8).init(gpa);
	265	try para.appendSlice(t.advanceLine());
	266
	267	var next_line = t.advanceLine();
	268	while (t.str.len > 0 and next_line.len > 0) {
	269	try para.append(' ');
	270	try para.appendSlice(next_line);
	271	next_line = t.advanceLine();
	272	}
	273
	274	try processInlineMarkdown(gpa, para.items, writer);
	275
	276	try writer.writeAll("\n</blockquote>\n");
	277	},
	278	}
	279	}
	280	}
	281
	282	pub fn nextInlineToken(src: []const u8) ?struct { InlineType, usize } {
	283	var pos: usize = 0;
	284	while (true) : (pos += 1) {
	285	pos = std.mem.indexOfAnyPos(u8, src, pos, "*`") orelse return null;
	286
	287	switch (src[pos]) {
	288	'*' => switch (src[pos + 1]) {
	289	'*' => return .{ .bold, pos },
	290	else => continue,
	291	},
	292	'`' => return .{ .code, pos },
	293	else => unreachable,
	294	}
	295	}
	296	}
	297
	298	fn processInlineMarkdown(gpa: std.mem.Allocator, src: []const u8, writer: anytype) !void {
	299	var bold = false;
	300	var remain = src;
	301
	302	while (nextInlineToken(remain)) \|tok\| {
	303	try writer.writeAll(remain[0..tok[1]]);
	304	remain = remain[tok[1]..];
	305	switch (tok[0]) {
	306	.bold => {
	307	bold = !bold;
	308	try writer.writeAll(switch (bold) {
	309	true => "<strong>",
	310	false => "</strong>",
	311	});
	312	remain = remain[2..];
	313	},
	314	.code => {
	315	const end = std.mem.indexOfScalarPos(u8, remain, 1, '`') orelse @panic("invalid code block");
	316	const code = remain[1..end];
	317	try writer.writeAll("<code>");
	318	try writeTokenizedCode(gpa, code, writer);
	319	try writer.writeAll("</code>");
	320	remain = remain[end + 1 ..];
	321	},
	322	}
	323	}
	324	try writer.writeAll(remain);
	325
	326	if (bold) {
	327	std.debug.panic("Unterminated Bold in '{s}'", .{src});
	328	}
	329	}
	330
	331	// TODO: Change this to use Tree-sitter, which would also make it very simple
	332	// to highlight JavaScript code in addition to the language.
	333	fn writeTokenizedCode(gpa: std.mem.Allocator, code: []const u8, writer: anytype) !void {
	334	const code_z = try gpa.dupeZ(u8, code);
	335	defer gpa.free(code_z);
	336
	337	var t = core.Tokenizer{ .code = code_z };
	338
	339	t.skipWhitespace();
	340	var last: usize = 0;
	341	while (true) {
	342	t.skipWhitespace();
	343	const tok = t.next() catch {
	344	t.pos += 1;
	345	continue;
	346	} orelse break;
	347	try writer.writeAll(t.code[last..tok.offset]);
	348	try writer.print("<span class='tok-{s}{s}'>{s}</span>", .{
	349	@tagName(tok.tag),
	350	if (tok.tag.isKeyword()) " kw" else "",
	351	t.code[tok.offset..t.pos],
	352	});
	353	last = t.pos;
	354	}
	355	}
	356
	357	fn writeAnsi(gpa: std.mem.Allocator, code: []const u8, writer: anytype) !void {
	358	var remain = code;
	359	var count: usize = 0;
	360	while (std.mem.indexOfScalar(u8, remain, '\x1b')) \|esc_i\| {
	361	try writer.writeAll(remain[0..esc_i]);
	362	switch (remain[esc_i + 1]) {
	363	'[' => switch (remain[esc_i + 2]) {
	364	'0'...'9' => {
	365	const end = std.mem.indexOfScalarPos(u8, remain, esc_i + 2, 'm').?;
	366	const int = std.fmt.parseInt(u8, remain[esc_i + 2 .. end], 10) catch unreachable;
	367	switch (int) {
	368	else => {
	369	try writer.print("<span class='ansi-{d}'>", .{int});
	370	count += 1;
	371	},
	372	0 => {
	373	for (0..count) \|_\| {
	374	try writer.writeAll("</span>");
	375	}
	376	count = 0;
	377	},
	378	}
	379	remain = remain[end + 1 ..];
	380	},
	381	else => \|x\| std.debug.panic("Unknown esc + 2: {d}", .{x}),
	382	},
	383	else => \|x\| std.debug.panic("Unknown esc + 1: {d}", .{x}),
	384	}
	385	}
	386	_ = gpa;
	387	}

src/manual/template.html created+134

...	...	@@ -0,0 +1,134 @@
	1	<!doctype html>
	2	<html lang="en">
	3	<head>
	4	<meta charset="UTF-8" />
	5	<meta name="viewport" content="width=device-width, initial-scale=1.0" />
	6	<title>Language Reference</title>
	7	<style>
	8	* {
	9	margin: 0;
	10	padding: 0;
	11	box-sizing: border-box;
	12	}
	13	body {
	14	background-color: #fefefe;
	15	font-family:
	16	system-ui,
	17	-apple-system,
	18	BlinkMacSystemFont,
	19	"Segoe UI",
	20	Roboto,
	21	Oxygen,
	22	Ubuntu,
	23	Cantarell,
	24	"Open Sans",
	25	"Helvetica Neue",
	26	sans-serif;
	27	max-width: 45rem;
	28	padding: 1rem;
	29	margin: auto;
	30	}
	31	h1,
	32	h2,
	33	h3,
	34	h4,
	35	h5,
	36	h6,
	37	p,
	38	ul,
	39	pre,
	40	blockquote,
	41	table {
	42	margin-bottom: 1rem;
	43	}
	44	ul {
	45	padding-left: 2rem;
	46	}
	47	li {
	48	margin-bottom: 0.25rem;
	49	}
	50	blockquote {
	51	background-color: #ddd;
	52	border-left: 4px solid black;
	53	padding-left: 16px;
	54	padding-top: 8px;
	55	padding-bottom: 8px;
	56	}
	57	pre > code {
	58	border: none;
	59	padding: 0;
	60	background-color: transparent;
	61	}
	62	code,
	63	pre {
	64	font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas,
	65	"Liberation Mono", "Courier New", monospace;
	66	font-size: 0.9rem;
	67	border: 1px solid #9ca0b0;
	68	background-color: #eff1f5;
	69	border-radius: 4px;
	70	padding: 2px 4px;
	71	}
	72	pre {
	73	padding: 0.75rem 0.75rem;
	74	}
	75	.term {
	76	border: 1px solid #11111b;
	77	background-color: #24273a;
	78	color: #cdd6f4;
	79	}
	80	table,
	81	th,
	82	td {
	83	border: 1px solid #9ca0b0;
	84	border-collapse: collapse;
	85	}
	86	th,
	87	td {
	88	padding: 0.5rem;
	89	}
	90	.ansi-1 {
	91	font-weight: bold;
	92	}
	93	.ansi-31 {
	94	color: #ed8796;
	95	}
	96	.ansi-32 {
	97	color: #a6e3a1;
	98	}
	99	.ansi-33 {
	100	color: #eed49f;
	101	}
	102	.ansi-34 {
	103	color: #8aadf4;
	104	}
	105	.ansi-35 {
	106	color: #f5c2e7;
	107	}
	108	.ansi-36 {
	109	color: #94e2d5;
	110	}
	111	.ansi-2 {
	112	color: #8087a2;
	113	}
	114	.kw {
	115	color: #8839ef;
	116	font-weight: bold;
	117	}
	118	.tok-string {
	119	color: #40a02b;
	120	}
	121	.tok-builtin_id {
	122	color: #d20f39;
	123	font-weight: bold;
	124	}
	125	.tok-number_int,
	126	.tok-number_float {
	127	color: #fe640b;
	128	}
	129	</style>
	130	</head>
	131	<body>
	132	<main>{contents}</main>
	133	</body>
	134	</html>