Commit 7612a14a3d00c0640bc4c2244dc4cfdc698f7464 | ix

author	chloe caruso <git@paperclover.net> 2024-05-31 04:56:12-07:00
committer	chloe caruso <git@paperclover.net> 2024-05-31 04:56:12-07:00
log	7612a14a3d00c0640bc4c2244dc4cfdc698f7464
tree	a645b076118013e76b36af50d5f21f2a6d393818
parent	55f919b5c0cd47d3b4282b6d85bc57f5f93f0357

add language reference

14 files changed, 1065 insertions(+), 119 deletions(-)

CHANGELOG.md
README.md
build.zig
doc
- langref.md
src
- core
- langref
  - generate-langref.zig
  - langref.template.html

CHANGELOG.md+4-1

...	...	@@ -1,13 +1,16 @@
1	1	## 0.1.3
2	2
3		- feat(sema): coersion: numeric_literal -> float
	3	- feat(sema): coersion: numeric\_literal -> float
4	4	- feat(sema): prevent passing non-runtime compatible types like `type` to opaque functions
5	5	- feat(transform): lhs of assignments now validated
6	6	- feat(transform): mark useless discard
	7	- feat(transform): parse string escapes properly
	8	- feat(link): print string escapes properly
7	9	- feat: implement the `comptime` keyword (many comptime ops are not implemented)
8	10	- fix(sema): properly coerce assignments
9	11	- fix(transform): empty block generated in top level code tripped sema assertion
10	12	- fix(link): emit parenthesis to preserve operator precedence
	13	- fix(link): when main code has no side effects, do not emit any functions
11	14
12	15	## 0.1.2
13	16

README.md+7-77

...	...	@@ -1,85 +1,15 @@
1	1	# New Language Project
2	2
3		This language is a mix of features from JavaScript, Rust, and Zig. Some notable features carried over from each include:
	3	This is a high-level, general purpose programming language. It is a mix of Rust, Zig, and JavaScript's great points.
4	4
5		- Rust - Expressive control flow that is easy to write.
6		- Zig - Execute any code at compile time, everything is an expression.
7		- JavaScript - Garbage collected values with a versitile type system.
	5	For a guide on how to use the language, please see the Language Reference document.
8	6
9		## Developing
	7	## Building
10	8
11		To compile a development build, one can use the provided `./build` or `.\build.cmd` scripts to download Zig and build the compiler. These scripts are effective aliases of `zig build`. A release build can be built by passing the `--release` flag to any of these commands.
	9	The primary dependency of the language SDK is a Zig compiler. If one is not available, the `./build` and `.\build.cmd` scripts can subsitute for running `zig build` on POSIX and Windows systems, respectively. With a Zig compiler, a release build can be produced in a single command:
12	10
13		## Standard Library
	11	zig build --release
14	12
15		Currently, there is no standard library. One will be introduced later.
	13	To run the standard packaging procedure, which builds the SDK for all officially supported targets:
16	14
17		## Hello World
18
19		```
20		let print = @js("console").log;
21		print("Hello, world!");
22		```
23
24		The `print` function is implemented by getting an external reference to the `console` JavaScript global, and then retriving the `log` property from it.
25
26		## Primative Data Types
27
28		\| name \| ts equivilent \| description \|
29		\| ---------- \| --------------------- \| ---------------------------------------------- \|
30		\| `int` \| `number` \| a number in signed 32-bit range \|
31		\| `float` \| `number` \| a 64-bit floating point number \|
32		\| `bool` \| `boolean` \| either `true` or `false` \|
33		\| `string` \| `string` \| a sequence of text characters \|
34		\| `void` \| `undefined` or `null` \| detectable lack of a value \|
35		\| `noreturn` \| `never` \| control flow prevents this value from existing \|
36
37		> note: `bigint` is planned but not added
38
39		## Variables
40
41		There are two types of variables, `let` and `var`.
42
43		`let` bindings cannot be overridden, and should be thought of as aliases of the expression within. Referring to these identifiers is equivilent to inlining it's where it was used, aside from side effect duplication.
44
45		```
46		let hello: int = 1;
47		let world = hello + 2;
48		let hello = hello * world;
49
50		@js("console").log(hello);
51		```
52
53		`var` bindings create mutable memory locations at runtime. This means they can be re-assigned to:
54
55		```
56		var message = "first message";
57
58		if @js("Math").random() > 0.5 {
59		message = "second message"
60		}
61
62		@js("console").log(message);
63		```
64
65		Using `let` is preferred whenever possible, as it will lead to more readable code, and the compiler is able to optimize it better. The above can be rewritten as:
66
67		```
68		let message = if @js("Math").random() > 0.5 {
69		"second message"
70		} else {
71		"first message"
72		};
73
74		@js("console").log(message);
75		```
76
77		It should be noted that numeric literals are a comptime-only data type, meaning it cannot be assigned to a `var` without an explicit type annotation. Since `int` is a strict subset of valid `float`s, the coersion will implicitly be a float, but a compiler warning will be emitted.
78
79		```
80		var number = 0; // `number` is a `float`, but not explicit
81		var number: int = 0; // `number` is `int`
82		var number: float = 0; // `number` is `float`
83		```
84
85		## Functions
	15	zig build package

build.zig+63-13

...	...	@@ -6,6 +6,7 @@ const OsVersion = Target.Query.OsVersion;
6	6	pub fn build(b: *std.Build) void {
7	7	b.reference_trace = 20;
8	8
	9	const native_target = b.resolveTargetQuery(.{});
9	10	const target = b.standardTargetOptions(.{});
10	11	const optimize_fallback = b.standardOptimizeOption(.{});
11	12	const optimize: std.builtin.OptimizeMode = switch (b.release_mode) {
...	...	@@ -18,13 +19,13 @@ pub fn build(b: *std.Build) void {
18	19	break :brk .ReleaseSafe;
19	20	},
20	21	};
21
22	22	const tracing = b.option(
23	23	TracingMode,
24	24	"trace",
25	25	"Enable tracing",
26	26	) orelse .none;
27	27
	28	const native_debug = buildWithTarget(b, native_target, .Debug, .none);
28	29	const main = buildWithTarget(b, target, optimize, tracing);
29	30
30	31	inline for (.{
...	...	@@ -33,17 +34,48 @@ pub fn build(b: *std.Build) void {
33	34	}) \|entry\| {
34	35	const artifact, const name = entry;
35	36
36		{
37		const run_cmd = b.addRunArtifact(artifact);
38		run_cmd.step.dependOn(b.getInstallStep());
39		if (b.args) \|args\| {
40		run_cmd.addArgs(args);
41		}
42		const run_step = b.step(name, "Run " ++ name ++ " binary");
43		run_step.dependOn(&run_cmd.step);
	37	const run_cmd = b.addRunArtifact(artifact);
	38	run_cmd.step.dependOn(b.getInstallStep());
	39	if (b.args) \|args\| {
	40	run_cmd.addArgs(args);
44	41	}
	42	const run_step = b.step(name, "Run " ++ name ++ " binary");
	43	run_step.dependOn(&run_cmd.step);
45	44	}
46	45
	46	const test_all = b.step("test", "Run all tests");
	47	inline for (.{
	48	.{ main.core_tests, "core" },
	49	}) \|entry\| {
	50	const artifact, const name = entry;
	51	const run_cmd = b.addRunArtifact(artifact);
	52	run_cmd.step.dependOn(b.getInstallStep());
	53	if (b.args) \|args\| {
	54	run_cmd.addArgs(args);
	55	}
	56	const run_step = b.step("test-" ++ name, "Test " ++ name ++ " module");
	57	run_step.dependOn(&run_cmd.step);
	58	test_all.dependOn(run_step);
	59	}
	60
	61	// Language Server
	62	const langref_generator = b.addExecutable(.{
	63	.name = "generate-langref",
	64	.root_source_file = b.path("src/langref/generate-langref.zig"),
	65	.target = native_target,
	66	.optimize = .Debug,
	67	.link_libc = true,
	68	});
	69	langref_generator.root_module.addImport("build_options", native_debug.opts_mod);
	70	langref_generator.root_module.addImport("core", native_debug.core);
	71	const langref_run = b.addRunArtifact(langref_generator);
	72	langref_run.addFileArg(native_debug.compiler.getEmittedBin());
	73	langref_run.addFileArg(b.path("doc/langref.md"));
	74	langref_run.addFileArg(b.path("src/langref/langref.template.html"));
	75	const langref_out = langref_run.addOutputFileArg("language-reference.html");
	76	const install_langref = b.addInstallFile(langref_out, "doc/langref.html");
	77	b.step("langref", "build language reference").dependOn(&install_langref.step);
	78
47	79	// const wasm_api = b.addExecutable(.{
48	80	// .name = "compiler",
49	81	// .root_source_file = b.path("src/wasm_api.zig"),
...	...	@@ -62,7 +94,8 @@ pub fn build(b: *std.Build) void {
62	94	// wasm_step.dependOn(&wasm_api.step);
63	95	// wasm_api.root_module.addImport("zg:PropsData", zg.module("PropsData"));
64	96
65		const release = b.step("release", "Package releases");
	97	// Packaging
	98	const package = b.step("package", "Package a release build");
66	99	const linux_version: OsVersion = .{ .semver = .{ .major = 1, .minor = 0, .patch = 0 } };
67	100	const mac_version: OsVersion = .{ .semver = .{ .major = 11, .minor = 0, .patch = 0 } };
68	101	const win_version: OsVersion = .{ .windows = .win7 };
...	...	@@ -88,10 +121,10 @@ pub fn build(b: *std.Build) void {
88	121
89	122	inline for (.{
90	123	.{ artifacts.compiler, "compiler" },
91		.{ artifacts.lsp, "lsp" },
	124	// .{ artifacts.lsp, "lsp" },
92	125	}) \|entry\| {
93	126	const artifact, const bin_name = entry;
94		release.dependOn(&b.addInstallFile(
	127	package.dependOn(&b.addInstallFile(
95	128	artifact.getEmittedBin(),
96	129	b.pathJoin(&.{
97	130	"release",
...	...	@@ -103,6 +136,9 @@ pub fn build(b: *std.Build) void {
103	136	}),
104	137	).step);
105	138	}
	139
	140	package.dependOn(&b.addInstallFile(b.path("CHANGELOG.md"), "release/CHANGELOG.md").step);
	141	package.dependOn(&b.addInstallFile(langref_out, "release/README.html").step);
106	142	}
107	143	}
108	144
...	...	@@ -110,8 +146,10 @@ const TracingMode = enum { none, spall, chrome, log };
110	146
111	147	const Artifacts = struct {
112	148	core: *Build.Module,
	149	core_tests: *Build.Step.Compile,
113	150	compiler: *Build.Step.Compile,
114	151	lsp: *Build.Step.Compile,
	152	opts_mod: *Build.Module,
115	153	};
116	154
117	155	fn buildWithTarget(
...	...	@@ -141,7 +179,17 @@ fn buildWithTarget(
141	179	core.addImport("tracer", tracer.module("tracer"));
142	180	core.addImport("zg:PropsData", zg.module("PropsData"));
143	181
144		// Compiler Testing Exe
	182	// Core Unit Tests
	183	const core_tests = b.addTest(.{
	184	.root_source_file = b.path("src/core/mod.zig"),
	185	.target = target,
	186	.optimize = optimize,
	187	});
	188	core_tests.root_module.addImport("build_options", opts_mod);
	189	core_tests.root_module.addImport("tracer", tracer.module("tracer"));
	190	core_tests.root_module.addImport("zg:PropsData", zg.module("PropsData"));
	191
	192	// Compiler Demo Exe
145	193	const compiler_exe = b.addExecutable(.{
146	194	.name = "compiler",
147	195	.root_source_file = b.path("src/core/main.zig"),
...	...	@@ -167,7 +215,9 @@ fn buildWithTarget(
167	215
168	216	return .{
169	217	.core = core,
	218	.core_tests = core_tests,
170	219	.compiler = compiler_exe,
171	220	.lsp = lsp_exe,
	221	.opts_mod = opts_mod,
172	222	};
173	223	}

doc/langref.md created+137

...	...	@@ -0,0 +1,137 @@
	1	# New Language Project
	2
	3	This is a high-level, general purpose programming language. It is a mix of Rust, Zig, and JavaScript's great points.
	4
	5	- Rust - Expressive control flow that is easy to write.
	6	- Zig - Execute any code at compile time, everything is an expression.
	7	- JavaScript - Garbage collected values with a versitile type system.
	8
	9	Key goals of this project are:
	10
	11	- Ease of Use: Use of type inference keeps code concise and easy to read and write.
	12	- Fast Iteration: The compiler is designed for quick incremental compilation and hot module reloading.
	13	- TODO: some more
	14
	15	## Standard Library
	16
	17	Currently, there is no standard library. One will be introduced later.
	18
	19	## Hello World
	20
	21	```lang:run
	22	let print = @js("console").log;
	23	print("Hello, world!");
	24	```
	25
	26	The `print` function is implemented by getting an external reference to the `console` JavaScript global, and then retriving the `log` property from it.
	27
	28	## Primative Data Types
	29
	30	\| name \| ts equivilent \| description \|
	31	\| ---------- \| --------------------- \| ---------------------------------------------- \|
	32	\| `int` \| `number` \| a number in signed 32-bit range \|
	33	\| `float` \| `number` \| a 64-bit floating point number \|
	34	\| `bool` \| `boolean` \| either `true` or `false` \|
	35	\| `string` \| `string` \| a sequence of text characters \|
	36	\| `void` \| `undefined` or `null` \| detectable lack of a value \|
	37	\| `noreturn` \| `never` \| control flow prevents this value from existing \|
	38
	39	> note: `bigint` is planned but not added
	40
	41	## Built-in Constants
	42
	43	In addition to the above types being built-in constants, the following are available.
	44
	45	\| name \| description \|
	46	\| ------- \| ------------------------------------- \|
	47	\| `true` \| boolean true (result of `1 == 1`) \|
	48	\| `false` \| boolean false (result of `1 == 2`) \|
	49	\| `std` \| handle to the standard library module \|
	50
	51	These constants are reserved names, and it is an error to re-declare them.
	52
	53	```lang:build
	54	let true = 1 == 0;
	55	```
	56
	57	## Variables
	58
	59	There are two types of variables, `let` and `var`.
	60
	61	`let` bindings cannot be overridden, and should be thought of as aliases of the expression within. Referring to these identifiers is equivilent to inlining it's where it was used, aside from side effect duplication.
	62
	63	```lang:run
	64	let hello: int = 1;
	65	let world = hello + 2;
	66	let hello = hello * world;
	67
	68	@js("console").log(hello);
	69	```
	70
	71	`var` bindings create mutable memory locations at runtime. This means they can be re-assigned to:
	72
	73	```lang:run
	74	var message = "first message";
	75
	76	if @js("Math").random() > 0.5 {
	77	message = "second message"
	78	}
	79
	80	@js("console").log(message);
	81	```
	82
	83	Using `let` is preferred whenever possible, as it will lead to more readable code, and the compiler is able to optimize it better. The above can be rewritten as:
	84
	85	```lang:run
	86	let message = if @js("Math").random() > 0.5 {
	87	"second message"
	88	} else {
	89	"first message"
	90	};
	91
	92	@js("console").log(message);
	93	```
	94
	95	It should be noted that numeric literals are a comptime-only data type, meaning it cannot be assigned to a `var` without an explicit type annotation. Since `int` is a strict subset of valid `float`s, the coersion will implicitly be a float, but a compiler warning will be emitted.
	96
	97	```lang
	98	var number = 0; // `number` is a `float`, but not explicit
	99	var number: int = 0; // `number` is `int`
	100	var number: float = 0; // `number` is `float`
	101	```
	102
	103	## Functions
	104
	105	Functions are declared with the `fn` keyword. Parameter types must be annotated. The function body is a block, which may use implicit return to return.
	106
	107	```lang:run
	108	fn add(a: int, b: int) -> int {
	109	a + b
	110	}
	111
	112	let print = @js("console").log;
	113	//print("Sum of 2 and 7" + add(2, 7));
	114	print(add(2, 7));
	115	```
	116
	117	> Bug: Return types are currently required. Omitting the `-> type` syntax currently makes the return type `void`, but this will be changed in the future to be an inferred return type.
	118
	119	The `return` expression can be used within functions to return early.
	120
	121	```lang
	122	fn something(a: int) -> bool {
	123	if a == 0 {
	124	return false;
	125	}
	126
	127	a > 100
	128	}
	129	```
	130
	131	## Number Literals
	132
	133	TODO
	134
	135	## String Literals
	136
	137	TODO

src/core/InternPool.zig+12-8

...	...	@@ -682,7 +682,7 @@ pub const Capture = packed struct(u32) {
682	682	start: u32,
683	683	len: u32,
684	684	pub fn get(slice: Slice, ip: *const InternPool) []Capture {
685		return @ptrCast(ip.extra.items[slice.start..][0..slice.len]);
	685	return @ptrCast(ip.extra.list.items[slice.start..][0..slice.len]);
686	686	}
687	687	};
688	688	};
...	...	@@ -1198,19 +1198,23 @@ const tracer = @import("tracer");
1198	1198
1199	1199	test InternPool {
1200	1200	const testing = std.testing;
1201		std.testing.refAllDeclsRecursive(@This());
	1201	const a = testing.allocator;
	1202
	1203	std.testing.refAllDecls(@This());
1202	1204
1203		var ip_struct: InternPool = .{ .gpa = std.testing.allocator };
	1205	var ip_struct: InternPool = .{};
1204	1206	const ip = &ip_struct;
1205		defer ip.deinit();
1206	1207
1207		const str = try ip.getOrPutString("hello", .no_embedded_nulls);
	1208	try ip.init(a);
	1209	defer ip.deinit(a);
	1210
	1211	const str = try ip.getOrPutString(a, "hello", .no_embedded_nulls);
1208	1212	try testing.expectEqualStrings("hello", str.toSlice(ip));
1209		const str2 = try ip.getOrPutString("hello", .no_embedded_nulls);
	1213	const str2 = try ip.getOrPutString(a, "hello", .no_embedded_nulls);
1210	1214	try testing.expectEqual(str, str2);
1211	1215
1212		const int1 = try ip.get(.{ .int32 = 21483 });
	1216	const int1 = try ip.get(a, .{ .int32 = 21483 });
1213	1217	try testing.expectEqual(@as(i32, 21483), ip.indexToKey(int1).int32);
1214		const int2 = try ip.get(.{ .int32 = 21483 });
	1218	const int2 = try ip.get(a, .{ .int32 = 21483 });
1215	1219	try testing.expectEqual(int1, int2);
1216	1220	}

src/core/JsLink.zig+132-8

...	...	@@ -71,13 +71,13 @@ pub fn linkJsCode(
71	71
72	72	try writer.print("// Language Compiler v0.1.3. Alpha quality software.\n", .{});
73	73
74		// 2. Print every function decl
75		for (decls[1..]) \|decl_index\| {
76		try link.printDecl(decl_index, writer);
77		}
78
79		// 3. Print the top-level code
80	74	if (root_decl.codegen.?.nodes.len > 1) {
	75	// 2. Print every function decl
	76	for (decls[1..]) \|decl_index\| {
	77	try link.printDecl(decl_index, writer);
	78	}
	79
	80	// 3. Print the top-level code
81	81	try link.printDeclContents(&root_decl.codegen.?, writer);
82	82	} else {
83	83	try writer.print("// (program has no observable effects, thus an empty file)\n", .{});
...	...	@@ -233,8 +233,7 @@ fn printExprLevel(
233	233	// instead of emitting void nodes.
234	234	.void => "undefined",
235	235	}),
236		// TODO: escapes
237		.string => \|val\| try writer.print("\"{s}\"", .{val}),
	236	.string => \|val\| try writeString(val, writer),
238	237
239	238	.comptime_float => unreachable, // was not lowered to float
240	239
...	...	@@ -389,6 +388,131 @@ fn printExprLevel(
389	388	}
390	389	}
391	390
	391	pub fn canPrintWithoutEscape(c: u21) bool {
	392	const first_ascii_printable = 0x20;
	393	const last_ascii_printable = 0x7E;
	394	const first_high_surrogate = 0xD800;
	395	const last_low_surrogate = 0xDFFF;
	396
	397	if (c <= last_ascii_printable) {
	398	return c >= first_ascii_printable and c != '\\' and c != '"';
	399	} else {
	400	return c != 0xFEFF and (c < first_high_surrogate or c > last_low_surrogate);
	401	}
	402	}
	403
	404	pub fn indexOfNeedsEscape(slice: []const u8) ?usize {
	405	var it = std.unicode.Wtf8Iterator{ .bytes = slice, .i = 0 };
	406	while (it.nextCodepoint()) \|c\| {
	407	if (!canPrintWithoutEscape(c))
	408	return it.i;
	409	}
	410	return null;
	411	}
	412
	413	// This function is loosely based off of Bun's `writeJSONString`, which
	414	// is a Zig port of esbuild's `printUnquotedUTF16` function.
	415	// Both projects are licensed under MIT
	416	fn writeString(decoded: []const u8, writer: anytype) !void {
	417	try writer.writeByte('"');
	418
	419	var text = decoded;
	420	while (text.len > 0) {
	421	const width = std.unicode.utf8ByteSequenceLength(text[0]) catch
	422	unreachable; // string is encoded incorrectly
	423
	424	const c: u21 = std.unicode.wtf8Decode(text[0..width]) catch
	425	unreachable; // string is encoded incorrectly
	426
	427	if (canPrintWithoutEscape(c)) {
	428	if (indexOfNeedsEscape(text)) \|j\| {
	429	try writer.writeAll(text[0..j]);
	430	text = text[j..];
	431	continue;
	432	} else {
	433	try writer.writeAll(text);
	434	break;
	435	}
	436	}
	437
	438	text = text[width..];
	439
	440	switch (c) {
	441	// Special-case the bell character since it may cause dumping this file to
	442	// the terminal to make a sound, which is undesirable. Note that we can't
	443	// use an octal literal to print this shorter since octal literals are not
	444	// allowed in strict mode (or in template strings).
	445	0x07 => {
	446	try writer.writeAll("\\x07");
	447	},
	448	0x08 => {
	449	try writer.writeAll("\\b");
	450	},
	451	0x0C => {
	452	try writer.writeAll("\\f");
	453	},
	454	'\n' => {
	455	try writer.writeAll("\\n");
	456	},
	457	std.ascii.control_code.cr => {
	458	try writer.writeAll("\\r");
	459	},
	460	std.ascii.control_code.vt => {
	461	try writer.writeAll("\\v");
	462	},
	463	'\\' => {
	464	try writer.writeAll("\\\\");
	465	},
	466	'"' => {
	467	try writer.writeAll("\\\"");
	468	},
	469	'\t' => {
	470	try writer.writeAll("\\t");
	471	},
	472
	473	else => {
	474	const hex_chars = "0123456789ABCDEF";
	475
	476	if (c < 0xFFFF) {
	477	const k = @as(usize, @intCast(c));
	478
	479	try writer.writeAll(&[_]u8{
	480	'\\',
	481	'u',
	482	hex_chars[(k >> 12) & 0xF],
	483	hex_chars[(k >> 8) & 0xF],
	484	hex_chars[(k >> 4) & 0xF],
	485	hex_chars[k & 0xF],
	486	});
	487	} else {
	488	const first_high_surrogate = 0xD800;
	489	const first_low_surrogate = 0xDC00;
	490
	491	const k = c - 0x10000;
	492	const lo = @as(usize, @intCast(first_high_surrogate + ((k >> 10) & 0x3FF)));
	493	const hi = @as(usize, @intCast(first_low_surrogate + (k & 0x3FF)));
	494
	495	try writer.writeAll(&[_]u8{
	496	'\\',
	497	'u',
	498	hex_chars[lo >> 12],
	499	hex_chars[(lo >> 8) & 15],
	500	hex_chars[(lo >> 4) & 15],
	501	hex_chars[lo & 15],
	502	'\\',
	503	'u',
	504	hex_chars[hi >> 12],
	505	hex_chars[(hi >> 8) & 15],
	506	hex_chars[(hi >> 4) & 15],
	507	hex_chars[hi & 15],
	508	});
	509	}
	510	},
	511	}
	512	}
	513	try writer.writeByte('"');
	514	}
	515
392	516	fn printLocal(link: *JsLink, local: JsTree.LocalIndex, writer: anytype) !void {
393	517	var t = tracer.trace(@src());
394	518	defer t.end();

src/core/Report.zig+9

...	...	@@ -416,6 +416,14 @@ const Prefix = enum {
416	416	.note => .blue,
417	417	};
418	418	}
	419
	420	fn symbol(prefix: Prefix, ascii: bool) []const u8 {
	421	return switch (prefix) {
	422	.@"error" => if (ascii) " X " else " ✘ ",
	423	.warn => if (ascii) " ! " else " ▲ ",
	424	.note => "",
	425	};
	426	}
419	427	};
420	428
421	429	pub fn deinit(report: Report, gpa: Allocator) void {
...	...	@@ -489,6 +497,7 @@ pub fn formatItem(
489	497
490	498	try tty.setColor(writer, prefix.color());
491	499	try tty.setColor(writer, .bold);
	500	try writer.writeAll(prefix.symbol(false));
492	501	try writer.writeAll(@tagName(prefix));
493	502	try tty.setColor(writer, .reset);
494	503	try tty.setColor(writer, .dim);

src/core/Tokenizer.zig+241-9

...	...	@@ -239,6 +239,12 @@ pub const Token = struct {
239	239	};
240	240	}
241	241
	242	pub fn isKeyword(tag: Tag) bool {
	243	return switch (tag) {
	244	inline else => \|t\| mem.startsWith(u8, @tagName(t), "kw_"),
	245	};
	246	}
	247
242	248	/// Most tokens have a static length, such as "struct" or "+="
243	249	pub fn staticLen(tag: Tag) ?u32 {
244	250	return if (tag.lexeme()) \|str\| @intCast(str.len) else null;
...	...	@@ -810,15 +816,18 @@ fn consumeString(t: *Tokenizer) !void {
810	816	'n' => t.pos += 2,
811	817	'r' => t.pos += 2,
812	818	't' => t.pos += 2,
813		'0' => switch (t.byteAt(2) orelse {
814		t.pos = start;
815		return error.UnterminatedString;
816		}) {
817		// '\0xFF' hex escape
818		'x' => {},
819		// '\0' is special cased
820		else => t.pos += 2,
	819	'x' => {
	820	switch (t.byteAt(2).?) {
	821	'0'...'9', 'a'...'f', 'A'...'F' => {},
	822	else => return error.InvalidEscapeSequence,
	823	}
	824	switch (t.byteAt(3).?) {
	825	'0'...'9', 'a'...'f', 'A'...'F' => {},
	826	else => return error.InvalidEscapeSequence,
	827	}
	828	t.pos += 4;
821	829	},
	830	'0' => t.pos += 2,
822	831	'u' => {
823	832	const esc_start = t.pos;
824	833	t.consumeStringUnicodeEscape() catch \|err\| switch (err) {
...	...	@@ -925,7 +934,11 @@ fn maybeUTF8ErrorAt(t: *Tokenizer, comptime real_err: anytype, at: u32) @Type(.{
925	934	_ = try t.codePoint();
926	935	return real_err;
927	936	},
928		else => return real_err,
	937	else => {
	938	if (@import("builtin").is_test)
	939	std.debug.print("{s}\n", .{@errorName(real_err)});
	940	return real_err;
	941	},
929	942	}
930	943	}
931	944
...	...	@@ -943,6 +956,8 @@ pub fn skipWhitespace(t: *Tokenizer) void {
943	956	}
944	957	}
945	958
	959	// end methods, start token-related utilities
	960
946	961	pub fn tokenLocation(
947	962	utf8_source: []const u8,
948	963	start_offset: ByteOffset,
...	...	@@ -985,6 +1000,169 @@ pub fn tokenLocation(
985	1000	return loc;
986	1001	}
987	1002
	1003	pub fn stringHasEscapes(esc: []const u8) bool {
	1004	return mem.indexOfScalar(u8, esc, '\\') != null;
	1005	}
	1006
	1007	/// Given a valid string literal contents (excluding the quotes), return
	1008	/// the length of the decoded string. This is helpful for pre-allocation.
	1009	pub fn decodeStringCount(esc: []const u8) usize {
	1010	var result_len: usize = 0;
	1011	var i: usize = 0;
	1012	while (mem.indexOfScalarPos(u8, esc, i, '\\')) \|backslash\| {
	1013	// add prior chars
	1014	result_len += backslash - i;
	1015
	1016	switch (esc[backslash + 1]) {
	1017	// single char escapes that add one ascii
	1018	'"', '\\', 'n', 'r', '0' => {
	1019	i = backslash + "\\n".len;
	1020	result_len += 1;
	1021	},
	1022	// hex literals can be distinguished only by reading
	1023	// the first char, classifying it in either two ranges
	1024	//
	1025	// - 0x00 thru 0x7F -> one byte
	1026	// - 0x80 thru 0xFF -> two bytes
	1027	//
	1028	// we check the second byte anyways to ensure safe modes
	1029	// catch potential invalid string literals.
	1030	'x' => switch (esc[backslash + 3]) {
	1031	'0'...'9', 'a'...'f', 'A'...'F' => {
	1032	i = backslash + "\\x00".len;
	1033	switch (esc[backslash + 2]) {
	1034	'0'...'7',
	1035	=> result_len += 1,
	1036
	1037	'8'...'9',
	1038	'a'...'f',
	1039	'A'...'F',
	1040	=> result_len += 2,
	1041
	1042	else => unreachable, // invalid hex string
	1043	}
	1044	},
	1045	else => unreachable, // invalid hex string
	1046	},
	1047	// unicode literals contain a numerical parse
	1048	'u' => {
	1049	assert(esc[backslash + 2] == '{');
	1050
	1051	// skip 0x hex prefix if present
	1052	const start = if (esc[backslash + 4] == 'x')
	1053	backslash + 5
	1054	else
	1055	backslash + 3;
	1056
	1057	const end = mem.indexOfScalarPos(u8, esc, start, '}') orelse
	1058	unreachable; // invalid escape, missing }
	1059	const slice = esc[start..end];
	1060	const code_point = std.fmt.parseInt(u21, slice, 16) catch
	1061	unreachable; // invalid numeric string
	1062	result_len += std.unicode.utf8CodepointSequenceLength(code_point) catch
	1063	unreachable; // invalid codepoint
	1064
	1065	i = end + 1;
	1066	},
	1067
	1068	else => unreachable, // invalid escape
	1069	}
	1070	}
	1071	return result_len + (esc.len - i);
	1072	}
	1073
	1074	/// Given a valid string literal contents (excluding the quotes), write
	1075	/// it's decoded form (WTF-8) into the buf.
	1076	pub fn decodeStringBuf(esc: []const u8, buf: []u8) []u8 {
	1077	var result_i: usize = 0;
	1078	var i: usize = 0;
	1079	while (mem.indexOfScalarPos(u8, esc, i, '\\')) \|backslash\| {
	1080	// add prior chars
	1081	@memcpy(buf[result_i..][0 .. backslash - i], esc[i..backslash]);
	1082	result_i += backslash - i;
	1083
	1084	switch (esc[backslash + 1]) {
	1085	'"' => {
	1086	i = backslash + "\\\"".len;
	1087	buf[result_i] = '"';
	1088	result_i += 1;
	1089	},
	1090	'\\' => {
	1091	i = backslash + "\\\\".len;
	1092	buf[result_i] = '\\';
	1093	result_i += 1;
	1094	},
	1095	'n' => {
	1096	i = backslash + "\\n".len;
	1097	buf[result_i] = '\n';
	1098	result_i += 1;
	1099	},
	1100	'r' => {
	1101	i = backslash + "\\r".len;
	1102	buf[result_i] = '\r';
	1103	result_i += 1;
	1104	},
	1105	'0' => {
	1106	i = backslash + "\\n".len;
	1107	buf[result_i] = 0;
	1108	result_i += 1;
	1109	},
	1110
	1111	'x' => {
	1112	const codepoint = std.fmt.parseInt(u8, esc[backslash + 2 ..][0..2], 16) catch
	1113	unreachable; // invalid string literal
	1114	i = backslash + "\\x00".len;
	1115	if (codepoint < 0x80) {
	1116	buf[result_i] = codepoint;
	1117	result_i += 1;
	1118	} else {
	1119	const len = std.unicode.wtf8Encode(codepoint, buf[result_i..][0..2]) catch
	1120	unreachable; // all codepoints <= 0xFF are valid
	1121	assert(len == 2);
	1122	result_i += 2;
	1123	}
	1124	},
	1125	// unicode literals contain a numerical parse
	1126	'u' => {
	1127	assert(esc[backslash + 2] == '{');
	1128	// skip 0x hex prefix if present
	1129	const start = if (esc[backslash + 4] == 'x')
	1130	backslash + 5
	1131	else
	1132	backslash + 3;
	1133	const end = mem.indexOfScalarPos(u8, esc, start, '}') orelse
	1134	unreachable; // invalid escape, missing }
	1135	const slice = esc[start..end];
	1136	const codepoint = std.fmt.parseInt(u21, slice, 16) catch
	1137	unreachable; // invalid numeric string
	1138	const len = std.unicode.utf8CodepointSequenceLength(codepoint) catch
	1139	unreachable; // invalid codepoint
	1140	const actual_len =
	1141	std.unicode.wtf8Encode(codepoint, buf[result_i..][0..len]) catch
	1142	unreachable; // already known to be valid codepoint
	1143	assert(len == actual_len);
	1144	result_i += len;
	1145	i = end + 1;
	1146	},
	1147
	1148	else => unreachable, // invalid escape
	1149	}
	1150	}
	1151	@memcpy(buf[result_i..][0 .. esc.len - i], esc[i..]);
	1152	return buf[0 .. result_i + esc.len - i];
	1153	}
	1154
	1155	/// Given a valid string literal contents (excluding the quotes),
	1156	/// decode it into a newly allocated slice.
	1157	pub fn decodeStringAlloc(alloc: mem.Allocator, esc: []const u8) ![]u8 {
	1158	const count = decodeStringCount(esc);
	1159	const memory = try alloc.alloc(u8, count);
	1160	const result = decodeStringBuf(esc, memory);
	1161	assert(result.ptr == memory.ptr);
	1162	assert(result.len == memory.len);
	1163	return memory;
	1164	}
	1165
988	1166	const Tokenizer = @This();
989	1167	const std = @import("std");
990	1168	const unicode = std.unicode;
...	...	@@ -993,3 +1171,57 @@ const mem = std.mem;
993	1171	const unicode_props = @import("unicode_props.zig");
994	1172	const T = @import("tracer");
995	1173	const Location = @import("Ast.zig").Location;
	1174
	1175	fn testStringDecode(expected: []const u8, escaped: []const u8) !void {
	1176	const testing = std.testing;
	1177	const decoded = try decodeStringAlloc(testing.allocator, escaped);
	1178	defer testing.allocator.free(decoded);
	1179	try testing.expectEqualSlices(u8, expected, decoded);
	1180	}
	1181
	1182	test "basic tokenizer example" {
	1183	const testing = std.testing;
	1184	try unicode_props.init();
	1185
	1186	// not valid source code, but testing the tokenization
	1187	const source =
	1188	\\var token = 1.2e9.2 4;
	1189	\\// comment ignored
	1190	\\pub fn hello(:+ true) id return
	1191	\\π @"hello" "hello \r\n \x1b"
	1192	;
	1193	const expected_tokens = [_]Token{
	1194	.{ .tag = .kw_var, .offset = 0 },
	1195	.{ .tag = .id, .offset = 4 },
	1196	.{ .tag = .equal, .offset = 10 },
	1197	.{ .tag = .number_float, .offset = 12 },
	1198	.{ .tag = .number_int, .offset = 20 },
	1199	.{ .tag = .semi, .offset = 21 },
	1200	.{ .tag = .kw_pub, .offset = 42 },
	1201	.{ .tag = .kw_fn, .offset = 46 },
	1202	.{ .tag = .id, .offset = 49 },
	1203	.{ .tag = .start_paren, .offset = 54 },
	1204	.{ .tag = .colon, .offset = 55 },
	1205	.{ .tag = .plus, .offset = 56 },
	1206	.{ .tag = .id, .offset = 58 },
	1207	.{ .tag = .end_paren, .offset = 62 },
	1208	.{ .tag = .id, .offset = 64 },
	1209	.{ .tag = .kw_return, .offset = 67 },
	1210	.{ .tag = .id, .offset = 74 },
	1211	.{ .tag = .id, .offset = 77 },
	1212	.{ .tag = .string, .offset = 86 },
	1213	};
	1214	var tokenizer = Tokenizer{ .code = source };
	1215	for (expected_tokens) \|tok\| {
	1216	tokenizer.skipWhitespace();
	1217	try testing.expectEqual(tok, (try tokenizer.next()).?);
	1218	}
	1219	tokenizer.skipWhitespace();
	1220	try testing.expectEqual(null, (try tokenizer.next()));
	1221	}
	1222
	1223	test "string escapes" {
	1224	try testStringDecode("hello world", "hello world");
	1225	try testStringDecode("hello\n\rw\x00or\x5ald", "hello\\n\\rw\\0or\\x5ald");
	1226	try testStringDecode("255: \xC3\xBF", "255: \\xFF");
	1227	}

src/core/Transform.zig+18-1

...	...	@@ -707,7 +707,7 @@ fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) Alloc
707	707
708	708	.string => {
709	709	const str = t.ast.basicNodeSlice(node);
710		const sliced = try t.addSlicedString(str[1 .. str.len - 1]);
	710	const sliced = try t.addDecodedSlicedString(str[1 .. str.len - 1]);
711	711	return try t.addInstRef(.{
712	712	.tag = .string,
713	713	.data = EncodedData.encode(.string, .{
...	...	@@ -1507,6 +1507,23 @@ fn addSlicedString(t: *Transform, str: []const u8) !SlicedString {
1507	1507	return string;
1508	1508	}
1509	1509
	1510	fn addDecodedSlicedString(t: *Transform, escaped: []const u8) !SlicedString {
	1511	const decoded = if (Tokenizer.stringHasEscapes(escaped)) brk: {
	1512	const len = Tokenizer.decodeStringCount(escaped);
	1513
	1514	// use the scratch arraylist as a temporary storage for
	1515	// decoding the string. this is done so that duplicate
	1516	// strings will get de-duplicated in string_bytes
	1517	//
	1518	// TODO: see if this is a useless optimization
	1519	try t.scratch.ensureUnusedCapacity(t.gpa, @divFloor(len, @sizeOf(u32)) + 1);
	1520	const buf = mem.sliceAsBytes(t.scratch.unusedCapacitySlice());
	1521
	1522	break :brk Tokenizer.decodeStringBuf(escaped, buf);
	1523	} else escaped;
	1524	return t.addSlicedString(decoded);
	1525	}
	1526
1510	1527	const Transform = @This();
1511	1528	const std = @import("std");
1512	1529	const mem = std.mem;

src/core/array.zig+1-1

...	...	@@ -271,7 +271,7 @@ pub fn ExtraStore(comptime set_kind: StoreKind, comptime set_mutability: Mutabil
271	271
272	272	pub fn writeU32(w: *TrailWriter, value: u32) void {
273	273	assert(w.leftover > 0);
274		w.ip.extra.appendAssumeCapacity(value);
	274	w.store.list.appendAssumeCapacity(value);
275	275	w.leftover -= 1;
276	276	}
277	277

src/core/main.zig+1-1

...	...	@@ -22,7 +22,7 @@ pub fn main() !void {
22	22
23	23	try unicode_props.init();
24	24
25		if (builtin.mode == .Debug) {
	25	if (builtin.mode == .Debug and std.io.getStdErr().isTty()) {
26	26	// zig build system doesnt print a newline :/
27	27	std.debug.print("\r\x1b[K", .{});
28	28	}

src/core/mod.zig+4

...	...	@@ -12,3 +12,7 @@ pub const Tokenizer = @import("Tokenizer.zig");
12	12	pub const Transform = @import("Transform.zig");
13	13	pub const unicode_props = @import("unicode_props.zig");
14	14	pub const util = @import("util.zig");
	15
	16	comptime {
	17	@import("std").testing.refAllDecls(@This());
	18	}

src/langref/generate-langref.zig created+358

...	...	@@ -0,0 +1,358 @@
	1	//! Language reference compiler. Converts markdown into HTML, using
	2	//! the Tokenizer to highlight code, and the compiler and node.js to
	3	//! show code results. Only ever built in Debug mode. We are not
	4	//! smart about memory. Written at 3am on a Thursday. Enjoy
	5	pub const std = @import("std");
	6	pub const core = @import("core");
	7
	8	const ArrayList = std.ArrayList;
	9
	10	pub fn main() !void {
	11	comptime std.debug.assert(@import("builtin").mode == .Debug);
	12	const gpa = std.heap.c_allocator;
	13
	14	var args = try std.process.argsWithAllocator(gpa);
	15	_ = args.next().?;
	16
	17	const compiler_path = args.next().?;
	18	const markdown_src = try std.fs.cwd().readFileAlloc(gpa, args.next().?, std.math.maxInt(usize));
	19	const template_src = try std.fs.cwd().readFileAlloc(gpa, args.next().?, std.math.maxInt(usize));
	20	const out_path = args.next().?;
	21
	22	var output = std.ArrayList(u8).init(gpa);
	23	defer output.deinit();
	24	try processMarkdown(
	25	gpa,
	26	markdown_src,
	27	compiler_path,
	28	output.writer(),
	29	);
	30
	31	const applied_template = try std.mem.replaceOwned(
	32	u8,
	33	gpa,
	34	template_src,
	35	"{contents}",
	36	output.items,
	37	);
	38	defer gpa.free(applied_template);
	39
	40	const out_file = try std.fs.createFileAbsolute(out_path, .{});
	41	defer out_file.close();
	42	try out_file.writeAll(applied_template);
	43	}
	44
	45	const BlockType = enum {
	46	para,
	47	h1,
	48	h2,
	49	h3,
	50	code_run,
	51	code_build,
	52	code,
	53	code_generic,
	54	ul,
	55	table,
	56	blockquote,
	57	};
	58
	59	const InlineType = enum {
	60	bold,
	61	code,
	62	};
	63
	64	const MarkdownTokenizer = struct {
	65	/// Remaining text
	66	str: []const u8,
	67
	68	fn skipWhitespace(t: *MarkdownTokenizer) void {
	69	t.str = std.mem.trimLeft(u8, t.str, " \r\n");
	70	}
	71
	72	fn advance(t: *MarkdownTokenizer, n: usize) []const u8 {
	73	defer {
	74	t.str = t.str[n..];
	75	}
	76	return t.str[0..n];
	77	}
	78
	79	fn advanceLine(t: *MarkdownTokenizer) []const u8 {
	80	return std.mem.trim(u8, t.advanceLineRaw(), " \r\n");
	81	}
	82
	83	fn advanceLineRaw(t: *MarkdownTokenizer) []const u8 {
	84	const n = std.mem.indexOfScalar(u8, t.str, '\n');
	85	const line = t.advance(if (n) \|x\| x + 1 else t.str.len);
	86	return line;
	87	}
	88
	89	fn blockType(t: *MarkdownTokenizer) BlockType {
	90	switch (t.str[0]) {
	91	'#' => {
	92	defer {
	93	while (t.str[0] == '#')
	94	_ = t.advance(1);
	95	}
	96
	97	if (t.str[1] != '#') return .h1;
	98	if (t.str[2] != '#') return .h2;
	99	if (t.str[3] != '#') return .h3;
	100	@panic("headers nested too deep");
	101	},
	102	'>' => {
	103	_ = t.advance(1);
	104	return .blockquote;
	105	},
	106	'\|' => {
	107	return .table;
	108	},
	109	'`' => {
	110	if (std.mem.startsWith(u8, t.str, "```")) {
	111	defer _ = t.advanceLine();
	112	if (std.mem.startsWith(u8, t.str, "```lang:run"))
	113	return .code_run;
	114	if (std.mem.startsWith(u8, t.str, "```lang:build"))
	115	return .code_build;
	116	if (std.mem.startsWith(u8, t.str, "```lang"))
	117	return .code;
	118	return .code_generic;
	119	}
	120	return .para;
	121	},
	122	'-' => {
	123	return .ul;
	124	},
	125	else => {
	126	return .para;
	127	},
	128	}
	129	}
	130	};
	131
	132	fn processMarkdown(gpa: std.mem.Allocator, src: []const u8, compiler_path: []const u8, writer: anytype) !void {
	133	var t = MarkdownTokenizer{ .str = src };
	134
	135	t.skipWhitespace();
	136	while (t.str.len > 0) : (t.skipWhitespace()) {
	137	switch (t.blockType()) {
	138	.para,
	139	=> {
	140	try writer.writeAll("<p>\n");
	141
	142	var para = ArrayList(u8).init(gpa);
	143	try para.appendSlice(t.advanceLine());
	144
	145	var next_line = t.advanceLine();
	146	while (t.str.len > 0 and next_line.len > 0) {
	147	try para.append(' ');
	148	try para.appendSlice(next_line);
	149	next_line = t.advanceLine();
	150	}
	151
	152	try processInlineMarkdown(gpa, para.items, writer);
	153
	154	try writer.writeAll("\n</p>\n");
	155	},
	156	.h1,
	157	.h2,
	158	.h3,
	159	=> \|tag\| {
	160	const html_tag = @tagName(tag);
	161	const name = t.advanceLine();
	162	try writer.print("<{0s}>{1s}</{0s}>\n", .{
	163	html_tag,
	164	name,
	165	});
	166	},
	167	.code_run,
	168	.code_build,
	169	.code,
	170	.code_generic,
	171	=> \|tag\| {
	172	var code = ArrayList(u8).init(gpa);
	173
	174	var next_line = t.advanceLineRaw();
	175	while (t.str.len > 0 and !std.mem.startsWith(u8, next_line, "```")) {
	176	try code.appendSlice(next_line);
	177	next_line = t.advanceLineRaw();
	178	}
	179
	180	try writer.writeAll("<pre><code>");
	181	try writeTokenizedCode(gpa, code.items, writer);
	182	try writer.writeAll("</code></pre>");
	183
	184	switch (tag) {
	185	.code_run,
	186	.code_build,
	187	=> {
	188	var build = std.process.Child.init(&.{ compiler_path, "/dev/stdin" }, gpa);
	189	build.stdin_behavior = .Pipe;
	190	build.stdout_behavior = .Pipe;
	191	build.stderr_behavior = .Pipe;
	192	var env_map = try std.process.getEnvMap(gpa);
	193	defer env_map.deinit();
	194	try env_map.put("YES_COLOR", "1");
	195	build.env_map = &env_map;
	196	try build.spawn();
	197	try build.stdin.?.writeAll(code.items);
	198	var out = ArrayList(u8).init(gpa);
	199	var err = ArrayList(u8).init(gpa);
	200	try build.collectOutput(&out, &err, std.math.maxInt(usize));
	201	const result = try build.wait();
	202	_ = result;
	203	if (std.mem.trim(u8, err.items, " \n\r").len > 0) {
	204	try writer.writeAll("<pre class='term'><code>");
	205	try writeAnsi(gpa, err.items, writer);
	206	try writer.writeAll("</code></pre>");
	207	}
	208	if (std.mem.trim(u8, out.items, " \n\r").len > 0) {
	209	try writer.writeAll("<pre><code>");
	210	try writer.writeAll(out.items);
	211	try writer.writeAll("</code></pre>");
	212	}
	213	},
	214	else => {},
	215	}
	216	},
	217	.ul,
	218	=> {
	219	try writer.writeAll("<ul>");
	220	while (t.str.len > 0 and (t.str[0] == '-')) {
	221	_ = t.advance(1);
	222	const line = t.advanceLine();
	223	try writer.writeAll("<li>");
	224	try processInlineMarkdown(gpa, line, writer);
	225	try writer.writeAll("</li>");
	226	}
	227	try writer.writeAll("</ul>");
	228	},
	229	.table,
	230	=> {
	231	_ = t.advanceLine();
	232	},
	233	.blockquote,
	234	=> {
	235	try writer.writeAll("<blockquote>\n");
	236
	237	var para = ArrayList(u8).init(gpa);
	238	try para.appendSlice(t.advanceLine());
	239
	240	var next_line = t.advanceLine();
	241	while (t.str.len > 0 and next_line.len > 0) {
	242	try para.append(' ');
	243	try para.appendSlice(next_line);
	244	next_line = t.advanceLine();
	245	}
	246
	247	try processInlineMarkdown(gpa, para.items, writer);
	248
	249	try writer.writeAll("\n</blockquote>\n");
	250	},
	251	}
	252	}
	253	}
	254
	255	pub fn nextInlineToken(src: []const u8) ?struct { InlineType, usize } {
	256	var pos: usize = 0;
	257	while (true) : (pos += 1) {
	258	pos = std.mem.indexOfAnyPos(u8, src, pos, "*`") orelse return null;
	259
	260	switch (src[pos]) {
	261	'*' => switch (src[pos + 1]) {
	262	'*' => return .{ .bold, pos },
	263	else => continue,
	264	},
	265	'`' => return .{ .code, pos },
	266	else => unreachable,
	267	}
	268	}
	269	}
	270
	271	fn processInlineMarkdown(gpa: std.mem.Allocator, src: []const u8, writer: anytype) !void {
	272	var bold = false;
	273	var remain = src;
	274
	275	while (nextInlineToken(remain)) \|tok\| {
	276	try writer.writeAll(remain[0..tok[1]]);
	277	remain = remain[tok[1]..];
	278	switch (tok[0]) {
	279	.bold => {
	280	bold = !bold;
	281	try writer.writeAll(switch (bold) {
	282	true => "<strong>",
	283	false => "</strong>",
	284	});
	285	remain = remain[2..];
	286	},
	287	.code => {
	288	const end = std.mem.indexOfScalarPos(u8, remain, 1, '`') orelse @panic("invalid code block");
	289	const code = remain[1..end];
	290	try writer.writeAll("<code>");
	291	try writeTokenizedCode(gpa, code, writer);
	292	try writer.writeAll("</code>");
	293	remain = remain[end + 1 ..];
	294	},
	295	}
	296	}
	297	try writer.writeAll(remain);
	298
	299	if (bold) {
	300	std.debug.panic("Unterminated Bold in '{s}'", .{src});
	301	}
	302	}
	303
	304	fn writeTokenizedCode(gpa: std.mem.Allocator, code: []const u8, writer: anytype) !void {
	305	const code_z = try gpa.dupeZ(u8, code);
	306	defer gpa.free(code_z);
	307
	308	var t = core.Tokenizer{ .code = code_z };
	309
	310	t.skipWhitespace();
	311	var last: usize = 0;
	312	while (true) {
	313	t.skipWhitespace();
	314	const tok = t.next() catch {
	315	t.pos += 1;
	316	continue;
	317	} orelse break;
	318	try writer.writeAll(t.code[last..tok.offset]);
	319	try writer.print("<span class='tok-{s}{s}'>{s}</span>", .{
	320	@tagName(tok.tag),
	321	if (tok.tag.isKeyword()) " kw" else "",
	322	t.code[tok.offset..t.pos],
	323	});
	324	last = t.pos;
	325	}
	326	}
	327
	328	fn writeAnsi(gpa: std.mem.Allocator, code: []const u8, writer: anytype) !void {
	329	var remain = code;
	330	var count: usize = 0;
	331	while (std.mem.indexOfScalar(u8, remain, '\x1b')) \|esc_i\| {
	332	try writer.writeAll(remain[0..esc_i]);
	333	switch (remain[esc_i + 1]) {
	334	'[' => switch (remain[esc_i + 2]) {
	335	'0'...'9' => {
	336	const end = std.mem.indexOfScalarPos(u8, remain, esc_i + 2, 'm').?;
	337	const int = std.fmt.parseInt(u8, remain[esc_i + 2 .. end], 10) catch unreachable;
	338	switch (int) {
	339	else => {
	340	try writer.print("<span class='ansi-{d}'>", .{int});
	341	count += 1;
	342	},
	343	0 => {
	344	for (0..count) \|_\| {
	345	try writer.writeAll("</span>");
	346	}
	347	count = 0;
	348	},
	349	}
	350	remain = remain[end + 1 ..];
	351	},
	352	else => \|x\| std.debug.panic("Unknown esc + 2: {d}", .{x}),
	353	},
	354	else => \|x\| std.debug.panic("Unknown esc + 1: {d}", .{x}),
	355	}
	356	}
	357	_ = gpa;
	358	}

src/langref/langref.template.html created+78

...	...	@@ -0,0 +1,78 @@
	1	<!doctype html>
	2	<html lang="en">
	3	<head>
	4	<meta charset="UTF-8">
	5	<meta name="viewport" content="width=device-width, initial-scale=1.0">
	6	<title>Language Reference</title>
	7	<style>
	8	* { margin: 0; padding: 0; box-sizing: border-box; }
	9	body {
	10	background-color: #fefefe;
	11	font-family: sans-serif;
	12	max-width: 45rem;
	13	padding: 1rem;
	14	margin: auto;
	15	}
	16	h1,h2,h3,h4,h5,h6,p,ul,pre,blockquote {
	17	margin-bottom: 1rem;
	18	}
	19	ul {
	20	padding-left: 2rem;
	21	}
	22	li {
	23	margin-bottom: 0.25rem;
	24	}
	25	blockquote {
	26	background-color: #ddd;
	27	border-left: 4px solid black;
	28	padding-left: 16px;
	29	padding-top: 8px;
	30	padding-bottom: 8px;
	31	}
	32	pre {
	33	padding: 0.5rem;
	34	}
	35	pre > code {
	36	border: none;
	37	padding: 0;
	38	background-color: transparent;
	39	}
	40	code, pre {
	41	border: 1px solid #999;
	42	background-color: #eeeeee;
	43	border-radius: 4px;
	44	padding: 2px 4px;
	45	}
	46	.term {
	47	border: 1px solid #333;
	48	background-color: #222;
	49	color:white;
	50	}
	51	.ansi-1 {
	52	font-weight: bold;
	53	}
	54	.ansi-31 {
	55	color:rgb(255, 68, 68);
	56	}
	57	.ansi-2 {
	58	color:#aaa;
	59	}
	60
	61	.kw {
	62	color: mediumpurple;
	63	font-weight: bold;
	64	}
	65	.tok-string {
	66	color: green;
	67	}
	68	.tok-number_int, .tok-number_float {
	69	color: orangered;
	70	}
	71	</style>
	72	</head>
	73	<body>
	74	<main>
	75	{contents}
	76	</main>
	77	</body>
	78	</html>