Commit 1822706c309b53326056cd1030df93696fe321c8 | ix

CHANGELOG.md+4-1

 @@ -1,13 +1,16 @@
 ## 0.1.3
-- feat(sema): coersion: numeric_literal -> float
+- feat(sema): coersion: numeric\_literal -> float
 - feat(sema): prevent passing non-runtime compatible types like `type` to opaque functions
 - feat(transform): lhs of assignments now validated
 - feat(transform): mark useless discard
+- feat(transform): parse string escapes properly
+- feat(link): print string escapes properly
 - feat: implement the `comptime` keyword (many comptime ops are not implemented)
 - fix(sema): properly coerce assignments
 - fix(transform): empty block generated in top level code tripped sema assertion
 - fix(link): emit parenthesis to preserve operator precedence
+- fix(link): when main code has no side effects, do not emit any functions
 ## 0.1.2

README.md+7-77

...	@@ -1,85 +1,15 @@	...	@@ -1,85 +1,15 @@
1	# New Language Project	1	# New Language Project
2		2
3	This language is a mix of features from JavaScript, Rust, and Zig. Some notable features carried over from each include:	3	This is a high-level, general purpose programming language. It is a mix of Rust, Zig, and JavaScript's great points.
4		4
5	- Rust - Expressive control flow that is easy to write.	5	For a guide on how to use the language, please see the Language Reference document.
6	- Zig - Execute any code at compile time, everything is an expression.
7	- JavaScript - Garbage collected values with a versitile type system.
8		6
9	## Developing	7	## Building
10		8
11	To compile a development build, one can use the provided `./build` or `.\build.cmd` scripts to download Zig and build the compiler. These scripts are effective aliases of `zig build`. A release build can be built by passing the `--release` flag to any of these commands.	9	The primary dependency of the language SDK is a Zig compiler. If one is not available, the `./build` and `.\build.cmd` scripts can subsitute for running `zig build` on POSIX and Windows systems, respectively. With a Zig compiler, a release build can be produced in a single command:
12		10
13	## Standard Library	11	zig build --release
14		12
15	Currently, there is no standard library. One will be introduced later.	13	To run the standard packaging procedure, which builds the SDK for all officially supported targets:
16		14
17	## Hello World	15	zig build package
18
19	```
20	let print = @js("console").log;
21	print("Hello, world!");
22	```
23
24	The `print` function is implemented by getting an external reference to the `console` JavaScript global, and then retriving the `log` property from it.
25
26	## Primative Data Types
27
28	\| name \| ts equivilent \| description \|
29	\| ---------- \| --------------------- \| ---------------------------------------------- \|
30	\| `int` \| `number` \| a number in signed 32-bit range \|
31	\| `float` \| `number` \| a 64-bit floating point number \|
32	\| `bool` \| `boolean` \| either `true` or `false` \|
33	\| `string` \| `string` \| a sequence of text characters \|
34	\| `void` \| `undefined` or `null` \| detectable lack of a value \|
35	\| `noreturn` \| `never` \| control flow prevents this value from existing \|
36
37	> note: `bigint` is planned but not added
38
39	## Variables
40
41	There are two types of variables, `let` and `var`.
42
43	`let` bindings cannot be overridden, and should be thought of as aliases of the expression within. Referring to these identifiers is equivilent to inlining it's where it was used, aside from side effect duplication.
44
45	```
46	let hello: int = 1;
47	let world = hello + 2;
48	let hello = hello * world;
49
50	@js("console").log(hello);
51	```
52
53	`var` bindings create mutable memory locations at runtime. This means they can be re-assigned to:
54
55	```
56	var message = "first message";
57
58	if @js("Math").random() > 0.5 {
59	message = "second message"
60	}
61
62	@js("console").log(message);
63	```
64
65	Using `let` is preferred whenever possible, as it will lead to more readable code, and the compiler is able to optimize it better. The above can be rewritten as:
66
67	```
68	let message = if @js("Math").random() > 0.5 {
69	"second message"
70	} else {
71	"first message"
72	};
73
74	@js("console").log(message);
75	```
76
77	It should be noted that numeric literals are a comptime-only data type, meaning it cannot be assigned to a `var` without an explicit type annotation. Since `int` is a strict subset of valid `float`s, the coersion will implicitly be a float, but a compiler warning will be emitted.
78
79	```
80	var number = 0; // `number` is a `float`, but not explicit
81	var number: int = 0; // `number` is `int`
82	var number: float = 0; // `number` is `float`
83	```
84
85	## Functions

build.zig+63-13

 @@ -6,6 +6,7 @@ const OsVersion = Target.Query.OsVersion;
 pub fn build(b: *std.Build) void {
     b.reference_trace = 20;
+    const native_target = b.resolveTargetQuery(.{});
     const target = b.standardTargetOptions(.{});
     const optimize_fallback = b.standardOptimizeOption(.{});
     const optimize: std.builtin.OptimizeMode = switch (b.release_mode) {
 @@ -18,13 +19,13 @@ pub fn build(b: *std.Build) void {
             break :brk .ReleaseSafe;
         },
     };
     const tracing = b.option(
         TracingMode,
         "trace",
         "Enable tracing",
     ) orelse .none;
+    const native_debug = buildWithTarget(b, native_target, .Debug, .none);
     const main = buildWithTarget(b, target, optimize, tracing);
     inline for (.{
 @@ -33,17 +34,48 @@ pub fn build(b: *std.Build) void {
     }) |entry| {
         const artifact, const name = entry;
-        {
+        const run_cmd = b.addRunArtifact(artifact);
-            const run_cmd = b.addRunArtifact(artifact);
+        run_cmd.step.dependOn(b.getInstallStep());
-            run_cmd.step.dependOn(b.getInstallStep());
+        if (b.args) |args| {
-            if (b.args) |args| {
+            run_cmd.addArgs(args);
-                run_cmd.addArgs(args);
-            }
-            const run_step = b.step(name, "Run " ++ name ++ " binary");
-            run_step.dependOn(&run_cmd.step);
         }
+        const run_step = b.step(name, "Run " ++ name ++ " binary");
+        run_step.dependOn(&run_cmd.step);
     }
+    const test_all = b.step("test", "Run all tests");
+    inline for (.{
+        .{ main.core_tests, "core" },
+    }) |entry| {
+        const artifact, const name = entry;
+        const run_cmd = b.addRunArtifact(artifact);
+        run_cmd.step.dependOn(b.getInstallStep());
+        if (b.args) |args| {
+            run_cmd.addArgs(args);
+        }
+        const run_step = b.step("test-" ++ name, "Test " ++ name ++ " module");
+        run_step.dependOn(&run_cmd.step);
+        test_all.dependOn(run_step);
+    }
+    // Language Server
+    const langref_generator = b.addExecutable(.{
+        .name = "generate-langref",
+        .root_source_file = b.path("src/langref/generate-langref.zig"),
+        .target = native_target,
+        .optimize = .Debug,
+        .link_libc = true,
+    });
+    langref_generator.root_module.addImport("build_options", native_debug.opts_mod);
+    langref_generator.root_module.addImport("core", native_debug.core);
+    const langref_run = b.addRunArtifact(langref_generator);
+    langref_run.addFileArg(native_debug.compiler.getEmittedBin());
+    langref_run.addFileArg(b.path("doc/langref.md"));
+    langref_run.addFileArg(b.path("src/langref/langref.template.html"));
+    const langref_out = langref_run.addOutputFileArg("language-reference.html");
+    const install_langref = b.addInstallFile(langref_out, "doc/langref.html");
+    b.step("langref", "build language reference").dependOn(&install_langref.step);
     // const wasm_api = b.addExecutable(.{
     //     .name = "compiler",
     //     .root_source_file = b.path("src/wasm_api.zig"),
 @@ -62,7 +94,8 @@ pub fn build(b: *std.Build) void {
     // wasm_step.dependOn(&wasm_api.step);
     // wasm_api.root_module.addImport("zg:PropsData", zg.module("PropsData"));
-    const release = b.step("release", "Package releases");
+    // Packaging
+    const package = b.step("package", "Package a release build");
     const linux_version: OsVersion = .{ .semver = .{ .major = 1, .minor = 0, .patch = 0 } };
     const mac_version: OsVersion = .{ .semver = .{ .major = 11, .minor = 0, .patch = 0 } };
     const win_version: OsVersion = .{ .windows = .win7 };
 @@ -88,10 +121,10 @@ pub fn build(b: *std.Build) void {
         inline for (.{
             .{ artifacts.compiler, "compiler" },
-            .{ artifacts.lsp, "lsp" },
+            // .{ artifacts.lsp, "lsp" },
         }) |entry| {
             const artifact, const bin_name = entry;
-            release.dependOn(&b.addInstallFile(
+            package.dependOn(&b.addInstallFile(
                 artifact.getEmittedBin(),
                 b.pathJoin(&.{
                     "release",
 @@ -103,6 +136,9 @@ pub fn build(b: *std.Build) void {
                 }),
             ).step);
         }
+        package.dependOn(&b.addInstallFile(b.path("CHANGELOG.md"), "release/CHANGELOG.md").step);
+        package.dependOn(&b.addInstallFile(langref_out, "release/README.html").step);
     }
 }
 @@ -110,8 +146,10 @@ const TracingMode = enum { none, spall, chrome, log };
 const Artifacts = struct {
     core: *Build.Module,
+    core_tests: *Build.Step.Compile,
     compiler: *Build.Step.Compile,
     lsp: *Build.Step.Compile,
+    opts_mod: *Build.Module,
 };
 fn buildWithTarget(
 @@ -141,7 +179,17 @@ fn buildWithTarget(
     core.addImport("tracer", tracer.module("tracer"));
     core.addImport("zg:PropsData", zg.module("PropsData"));
-    // Compiler Testing Exe
+    // Core Unit Tests
+    const core_tests = b.addTest(.{
+        .root_source_file = b.path("src/core/mod.zig"),
+        .target = target,
+        .optimize = optimize,
+    });
+    core_tests.root_module.addImport("build_options", opts_mod);
+    core_tests.root_module.addImport("tracer", tracer.module("tracer"));
+    core_tests.root_module.addImport("zg:PropsData", zg.module("PropsData"));
+    // Compiler Demo Exe
     const compiler_exe = b.addExecutable(.{
         .name = "compiler",
         .root_source_file = b.path("src/core/main.zig"),
 @@ -167,7 +215,9 @@ fn buildWithTarget(
     return .{
         .core = core,
+        .core_tests = core_tests,
         .compiler = compiler_exe,
         .lsp = lsp_exe,
+        .opts_mod = opts_mod,
     };
 }

doc/langref.md created+137

...	@@ -0,0 +1,137 @@
1	# New Language Project
2
3	This is a high-level, general purpose programming language. It is a mix of Rust, Zig, and JavaScript's great points.
4
5	- Rust - Expressive control flow that is easy to write.
6	- Zig - Execute any code at compile time, everything is an expression.
7	- JavaScript - Garbage collected values with a versitile type system.
8
9	Key goals of this project are:
10
11	- Ease of Use: Use of type inference keeps code concise and easy to read and write.
12	- Fast Iteration: The compiler is designed for quick incremental compilation and hot module reloading.
13	- TODO: some more
14
15	## Standard Library
16
17	Currently, there is no standard library. One will be introduced later.
18
19	## Hello World
20
21	```lang:run
22	let print = @js("console").log;
23	print("Hello, world!");
24	```
25
26	The `print` function is implemented by getting an external reference to the `console` JavaScript global, and then retriving the `log` property from it.
27
28	## Primative Data Types
29
30	\| name \| ts equivilent \| description \|
31	\| ---------- \| --------------------- \| ---------------------------------------------- \|
32	\| `int` \| `number` \| a number in signed 32-bit range \|
33	\| `float` \| `number` \| a 64-bit floating point number \|
34	\| `bool` \| `boolean` \| either `true` or `false` \|
35	\| `string` \| `string` \| a sequence of text characters \|
36	\| `void` \| `undefined` or `null` \| detectable lack of a value \|
37	\| `noreturn` \| `never` \| control flow prevents this value from existing \|
38
39	> note: `bigint` is planned but not added
40
41	## Built-in Constants
42
43	In addition to the above types being built-in constants, the following are available.
44
45	\| name \| description \|
46	\| ------- \| ------------------------------------- \|
47	\| `true` \| boolean true (result of `1 == 1`) \|
48	\| `false` \| boolean false (result of `1 == 2`) \|
49	\| `std` \| handle to the standard library module \|
50
51	These constants are reserved names, and it is an error to re-declare them.
52
53	```lang:build
54	let true = 1 == 0;
55	```
56
57	## Variables
58
59	There are two types of variables, `let` and `var`.
60
61	`let` bindings cannot be overridden, and should be thought of as aliases of the expression within. Referring to these identifiers is equivilent to inlining it's where it was used, aside from side effect duplication.
62
63	```lang:run
64	let hello: int = 1;
65	let world = hello + 2;
66	let hello = hello * world;
67
68	@js("console").log(hello);
69	```
70
71	`var` bindings create mutable memory locations at runtime. This means they can be re-assigned to:
72
73	```lang:run
74	var message = "first message";
75
76	if @js("Math").random() > 0.5 {
77	message = "second message"
78	}
79
80	@js("console").log(message);
81	```
82
83	Using `let` is preferred whenever possible, as it will lead to more readable code, and the compiler is able to optimize it better. The above can be rewritten as:
84
85	```lang:run
86	let message = if @js("Math").random() > 0.5 {
87	"second message"
88	} else {
89	"first message"
90	};
91
92	@js("console").log(message);
93	```
94
95	It should be noted that numeric literals are a comptime-only data type, meaning it cannot be assigned to a `var` without an explicit type annotation. Since `int` is a strict subset of valid `float`s, the coersion will implicitly be a float, but a compiler warning will be emitted.
96
97	```lang
98	var number = 0; // `number` is a `float`, but not explicit
99	var number: int = 0; // `number` is `int`
100	var number: float = 0; // `number` is `float`
101	```
102
103	## Functions
104
105	Functions are declared with the `fn` keyword. Parameter types must be annotated. The function body is a block, which may use implicit return to return.
106
107	```lang:run
108	fn add(a: int, b: int) -> int {
109	a + b
110	}
111
112	let print = @js("console").log;
113	//print("Sum of 2 and 7" + add(2, 7));
114	print(add(2, 7));
115	```
116
117	> Bug: Return types are currently required. Omitting the `-> type` syntax currently makes the return type `void`, but this will be changed in the future to be an inferred return type.
118
119	The `return` expression can be used within functions to return early.
120
121	```lang
122	fn something(a: int) -> bool {
123	if a == 0 {
124	return false;
125	}
126
127	a > 100
128	}
129	```
130
131	## Number Literals
132
133	TODO
134
135	## String Literals
136
137	TODO

src/core/InternPool.zig+12-8

 @@ -682,7 +682,7 @@ pub const Capture = packed struct(u32) {
         start: u32,
         len: u32,
         pub fn get(slice: Slice, ip: *const InternPool) []Capture {
-            return @ptrCast(ip.extra.items[slice.start..][0..slice.len]);
+            return @ptrCast(ip.extra.list.items[slice.start..][0..slice.len]);
         }
     };
 };
 @@ -1198,19 +1198,23 @@ const tracer = @import("tracer");
 test InternPool {
     const testing = std.testing;
-    std.testing.refAllDeclsRecursive(@This());
+    const a = testing.allocator;
+    std.testing.refAllDecls(@This());
-    var ip_struct: InternPool = .{ .gpa = std.testing.allocator };
+    var ip_struct: InternPool = .{};
     const ip = &ip_struct;
-    defer ip.deinit();
-    const str = try ip.getOrPutString("hello", .no_embedded_nulls);
+    try ip.init(a);
+    defer ip.deinit(a);
+    const str = try ip.getOrPutString(a, "hello", .no_embedded_nulls);
     try testing.expectEqualStrings("hello", str.toSlice(ip));
-    const str2 = try ip.getOrPutString("hello", .no_embedded_nulls);
+    const str2 = try ip.getOrPutString(a, "hello", .no_embedded_nulls);
     try testing.expectEqual(str, str2);
-    const int1 = try ip.get(.{ .int32 = 21483 });
+    const int1 = try ip.get(a, .{ .int32 = 21483 });
     try testing.expectEqual(@as(i32, 21483), ip.indexToKey(int1).int32);
-    const int2 = try ip.get(.{ .int32 = 21483 });
+    const int2 = try ip.get(a, .{ .int32 = 21483 });
     try testing.expectEqual(int1, int2);
 }

src/core/JsLink.zig+132-8

...	@@ -71,13 +71,13 @@ pub fn linkJsCode(	...	@@ -71,13 +71,13 @@ pub fn linkJsCode(
71		71
72	try writer.print("// Language Compiler v0.1.3. Alpha quality software.\n", .{});	72	try writer.print("// Language Compiler v0.1.3. Alpha quality software.\n", .{});
73		73
74	// 2. Print every function decl
75	for (decls[1..]) \|decl_index\| {
76	try link.printDecl(decl_index, writer);
77	}
78
79	// 3. Print the top-level code
80	if (root_decl.codegen.?.nodes.len > 1) {	74	if (root_decl.codegen.?.nodes.len > 1) {
		75	// 2. Print every function decl
		76	for (decls[1..]) \|decl_index\| {
		77	try link.printDecl(decl_index, writer);
		78	}
		79
		80	// 3. Print the top-level code
81	try link.printDeclContents(&root_decl.codegen.?, writer);	81	try link.printDeclContents(&root_decl.codegen.?, writer);
82	} else {	82	} else {
83	try writer.print("// (program has no observable effects, thus an empty file)\n", .{});	83	try writer.print("// (program has no observable effects, thus an empty file)\n", .{});
...	@@ -233,8 +233,7 @@ fn printExprLevel(	...	@@ -233,8 +233,7 @@ fn printExprLevel(
233	// instead of emitting void nodes.	233	// instead of emitting void nodes.
234	.void => "undefined",	234	.void => "undefined",
235	}),	235	}),
236	// TODO: escapes	236	.string => \|val\| try writeString(val, writer),
237	.string => \|val\| try writer.print("\"{s}\"", .{val}),
238		237
239	.comptime_float => unreachable, // was not lowered to float	238	.comptime_float => unreachable, // was not lowered to float
240		239
...	@@ -389,6 +388,131 @@ fn printExprLevel(	...	@@ -389,6 +388,131 @@ fn printExprLevel(
389	}	388	}
390	}	389	}
391		390
		391	pub fn canPrintWithoutEscape(c: u21) bool {
		392	const first_ascii_printable = 0x20;
		393	const last_ascii_printable = 0x7E;
		394	const first_high_surrogate = 0xD800;
		395	const last_low_surrogate = 0xDFFF;
		396
		397	if (c <= last_ascii_printable) {
		398	return c >= first_ascii_printable and c != '\\' and c != '"';
		399	} else {
		400	return c != 0xFEFF and (c < first_high_surrogate or c > last_low_surrogate);
		401	}
		402	}
		403
		404	pub fn indexOfNeedsEscape(slice: []const u8) ?usize {
		405	var it = std.unicode.Wtf8Iterator{ .bytes = slice, .i = 0 };
		406	while (it.nextCodepoint()) \|c\| {
		407	if (!canPrintWithoutEscape(c))
		408	return it.i;
		409	}
		410	return null;
		411	}
		412
		413	// This function is loosely based off of Bun's `writeJSONString`, which
		414	// is a Zig port of esbuild's `printUnquotedUTF16` function.
		415	// Both projects are licensed under MIT
		416	fn writeString(decoded: []const u8, writer: anytype) !void {
		417	try writer.writeByte('"');
		418
		419	var text = decoded;
		420	while (text.len > 0) {
		421	const width = std.unicode.utf8ByteSequenceLength(text[0]) catch
		422	unreachable; // string is encoded incorrectly
		423
		424	const c: u21 = std.unicode.wtf8Decode(text[0..width]) catch
		425	unreachable; // string is encoded incorrectly
		426
		427	if (canPrintWithoutEscape(c)) {
		428	if (indexOfNeedsEscape(text)) \|j\| {
		429	try writer.writeAll(text[0..j]);
		430	text = text[j..];
		431	continue;
		432	} else {
		433	try writer.writeAll(text);
		434	break;
		435	}
		436	}
		437
		438	text = text[width..];
		439
		440	switch (c) {
		441	// Special-case the bell character since it may cause dumping this file to
		442	// the terminal to make a sound, which is undesirable. Note that we can't
		443	// use an octal literal to print this shorter since octal literals are not
		444	// allowed in strict mode (or in template strings).
		445	0x07 => {
		446	try writer.writeAll("\\x07");
		447	},
		448	0x08 => {
		449	try writer.writeAll("\\b");
		450	},
		451	0x0C => {
		452	try writer.writeAll("\\f");
		453	},
		454	'\n' => {
		455	try writer.writeAll("\\n");
		456	},
		457	std.ascii.control_code.cr => {
		458	try writer.writeAll("\\r");
		459	},
		460	std.ascii.control_code.vt => {
		461	try writer.writeAll("\\v");
		462	},
		463	'\\' => {
		464	try writer.writeAll("\\\\");
		465	},
		466	'"' => {
		467	try writer.writeAll("\\\"");
		468	},
		469	'\t' => {
		470	try writer.writeAll("\\t");
		471	},
		472
		473	else => {
		474	const hex_chars = "0123456789ABCDEF";
		475
		476	if (c < 0xFFFF) {
		477	const k = @as(usize, @intCast(c));
		478
		479	try writer.writeAll(&[_]u8{
		480	'\\',
		481	'u',
		482	hex_chars[(k >> 12) & 0xF],
		483	hex_chars[(k >> 8) & 0xF],
		484	hex_chars[(k >> 4) & 0xF],
		485	hex_chars[k & 0xF],
		486	});
		487	} else {
		488	const first_high_surrogate = 0xD800;
		489	const first_low_surrogate = 0xDC00;
		490
		491	const k = c - 0x10000;
		492	const lo = @as(usize, @intCast(first_high_surrogate + ((k >> 10) & 0x3FF)));
		493	const hi = @as(usize, @intCast(first_low_surrogate + (k & 0x3FF)));
		494
		495	try writer.writeAll(&[_]u8{
		496	'\\',
		497	'u',
		498	hex_chars[lo >> 12],
		499	hex_chars[(lo >> 8) & 15],
		500	hex_chars[(lo >> 4) & 15],
		501	hex_chars[lo & 15],
		502	'\\',
		503	'u',
		504	hex_chars[hi >> 12],
		505	hex_chars[(hi >> 8) & 15],
		506	hex_chars[(hi >> 4) & 15],
		507	hex_chars[hi & 15],
		508	});
		509	}
		510	},
		511	}
		512	}
		513	try writer.writeByte('"');
		514	}
		515
392	fn printLocal(link: *JsLink, local: JsTree.LocalIndex, writer: anytype) !void {	516	fn printLocal(link: *JsLink, local: JsTree.LocalIndex, writer: anytype) !void {
393	var t = tracer.trace(@src());	517	var t = tracer.trace(@src());
394	defer t.end();	518	defer t.end();

src/core/Report.zig+9

 @@ -416,6 +416,14 @@ const Prefix = enum {
             .note => .blue,
         };
     }
+    fn symbol(prefix: Prefix, ascii: bool) []const u8 {
+        return switch (prefix) {
+            .@"error" => if (ascii) " X " else " ✘ ",
+            .warn => if (ascii) " ! " else " ▲ ",
+            .note => "",
+        };
+    }
 };
 pub fn deinit(report: Report, gpa: Allocator) void {
 @@ -489,6 +497,7 @@ pub fn formatItem(
         try tty.setColor(writer, prefix.color());
         try tty.setColor(writer, .bold);
+        try writer.writeAll(prefix.symbol(false));
         try writer.writeAll(@tagName(prefix));
         try tty.setColor(writer, .reset);
         try tty.setColor(writer, .dim);

src/core/Tokenizer.zig+241-9

...	@@ -239,6 +239,12 @@ pub const Token = struct {	...	@@ -239,6 +239,12 @@ pub const Token = struct {
239	};	239	};
240	}	240	}
241		241
		242	pub fn isKeyword(tag: Tag) bool {
		243	return switch (tag) {
		244	inline else => \|t\| mem.startsWith(u8, @tagName(t), "kw_"),
		245	};
		246	}
		247
242	/// Most tokens have a static length, such as "struct" or "+="	248	/// Most tokens have a static length, such as "struct" or "+="
243	pub fn staticLen(tag: Tag) ?u32 {	249	pub fn staticLen(tag: Tag) ?u32 {
244	return if (tag.lexeme()) \|str\| @intCast(str.len) else null;	250	return if (tag.lexeme()) \|str\| @intCast(str.len) else null;
...	@@ -810,15 +816,18 @@ fn consumeString(t: *Tokenizer) !void {	...	@@ -810,15 +816,18 @@ fn consumeString(t: *Tokenizer) !void {
810	'n' => t.pos += 2,	816	'n' => t.pos += 2,
811	'r' => t.pos += 2,	817	'r' => t.pos += 2,
812	't' => t.pos += 2,	818	't' => t.pos += 2,
813	'0' => switch (t.byteAt(2) orelse {	819	'x' => {
814	t.pos = start;	820	switch (t.byteAt(2).?) {
815	return error.UnterminatedString;	821	'0'...'9', 'a'...'f', 'A'...'F' => {},
816	}) {	822	else => return error.InvalidEscapeSequence,
817	// '\0xFF' hex escape	823	}
818	'x' => {},	824	switch (t.byteAt(3).?) {
819	// '\0' is special cased	825	'0'...'9', 'a'...'f', 'A'...'F' => {},
820	else => t.pos += 2,	826	else => return error.InvalidEscapeSequence,
		827	}
		828	t.pos += 4;
821	},	829	},
		830	'0' => t.pos += 2,
822	'u' => {	831	'u' => {
823	const esc_start = t.pos;	832	const esc_start = t.pos;
824	t.consumeStringUnicodeEscape() catch \|err\| switch (err) {	833	t.consumeStringUnicodeEscape() catch \|err\| switch (err) {
...	@@ -925,7 +934,11 @@ fn maybeUTF8ErrorAt(t: *Tokenizer, comptime real_err: anytype, at: u32) @Type(.{	...	@@ -925,7 +934,11 @@ fn maybeUTF8ErrorAt(t: *Tokenizer, comptime real_err: anytype, at: u32) @Type(.{
925	_ = try t.codePoint();	934	_ = try t.codePoint();
926	return real_err;	935	return real_err;
927	},	936	},
928	else => return real_err,	937	else => {
		938	if (@import("builtin").is_test)
		939	std.debug.print("{s}\n", .{@errorName(real_err)});
		940	return real_err;
		941	},
929	}	942	}
930	}	943	}
931		944
...	@@ -943,6 +956,8 @@ pub fn skipWhitespace(t: *Tokenizer) void {	...	@@ -943,6 +956,8 @@ pub fn skipWhitespace(t: *Tokenizer) void {
943	}	956	}
944	}	957	}
945		958
		959	// end methods, start token-related utilities
		960
946	pub fn tokenLocation(	961	pub fn tokenLocation(
947	utf8_source: []const u8,	962	utf8_source: []const u8,
948	start_offset: ByteOffset,	963	start_offset: ByteOffset,
...	@@ -985,6 +1000,169 @@ pub fn tokenLocation(	...	@@ -985,6 +1000,169 @@ pub fn tokenLocation(
985	return loc;	1000	return loc;
986	}	1001	}
987		1002
		1003	pub fn stringHasEscapes(esc: []const u8) bool {
		1004	return mem.indexOfScalar(u8, esc, '\\') != null;
		1005	}
		1006
		1007	/// Given a valid string literal contents (excluding the quotes), return
		1008	/// the length of the decoded string. This is helpful for pre-allocation.
		1009	pub fn decodeStringCount(esc: []const u8) usize {
		1010	var result_len: usize = 0;
		1011	var i: usize = 0;
		1012	while (mem.indexOfScalarPos(u8, esc, i, '\\')) \|backslash\| {
		1013	// add prior chars
		1014	result_len += backslash - i;
		1015
		1016	switch (esc[backslash + 1]) {
		1017	// single char escapes that add one ascii
		1018	'"', '\\', 'n', 'r', '0' => {
		1019	i = backslash + "\\n".len;
		1020	result_len += 1;
		1021	},
		1022	// hex literals can be distinguished only by reading
		1023	// the first char, classifying it in either two ranges
		1024	//
		1025	// - 0x00 thru 0x7F -> one byte
		1026	// - 0x80 thru 0xFF -> two bytes
		1027	//
		1028	// we check the second byte anyways to ensure safe modes
		1029	// catch potential invalid string literals.
		1030	'x' => switch (esc[backslash + 3]) {
		1031	'0'...'9', 'a'...'f', 'A'...'F' => {
		1032	i = backslash + "\\x00".len;
		1033	switch (esc[backslash + 2]) {
		1034	'0'...'7',
		1035	=> result_len += 1,
		1036
		1037	'8'...'9',
		1038	'a'...'f',
		1039	'A'...'F',
		1040	=> result_len += 2,
		1041
		1042	else => unreachable, // invalid hex string
		1043	}
		1044	},
		1045	else => unreachable, // invalid hex string
		1046	},
		1047	// unicode literals contain a numerical parse
		1048	'u' => {
		1049	assert(esc[backslash + 2] == '{');
		1050
		1051	// skip 0x hex prefix if present
		1052	const start = if (esc[backslash + 4] == 'x')
		1053	backslash + 5
		1054	else
		1055	backslash + 3;
		1056
		1057	const end = mem.indexOfScalarPos(u8, esc, start, '}') orelse
		1058	unreachable; // invalid escape, missing }
		1059	const slice = esc[start..end];
		1060	const code_point = std.fmt.parseInt(u21, slice, 16) catch
		1061	unreachable; // invalid numeric string
		1062	result_len += std.unicode.utf8CodepointSequenceLength(code_point) catch
		1063	unreachable; // invalid codepoint
		1064
		1065	i = end + 1;
		1066	},
		1067
		1068	else => unreachable, // invalid escape
		1069	}
		1070	}
		1071	return result_len + (esc.len - i);
		1072	}
		1073
		1074	/// Given a valid string literal contents (excluding the quotes), write
		1075	/// it's decoded form (WTF-8) into the buf.
		1076	pub fn decodeStringBuf(esc: []const u8, buf: []u8) []u8 {
		1077	var result_i: usize = 0;
		1078	var i: usize = 0;
		1079	while (mem.indexOfScalarPos(u8, esc, i, '\\')) \|backslash\| {
		1080	// add prior chars
		1081	@memcpy(buf[result_i..][0 .. backslash - i], esc[i..backslash]);
		1082	result_i += backslash - i;
		1083
		1084	switch (esc[backslash + 1]) {
		1085	'"' => {
		1086	i = backslash + "\\\"".len;
		1087	buf[result_i] = '"';
		1088	result_i += 1;
		1089	},
		1090	'\\' => {
		1091	i = backslash + "\\\\".len;
		1092	buf[result_i] = '\\';
		1093	result_i += 1;
		1094	},
		1095	'n' => {
		1096	i = backslash + "\\n".len;
		1097	buf[result_i] = '\n';
		1098	result_i += 1;
		1099	},
		1100	'r' => {
		1101	i = backslash + "\\r".len;
		1102	buf[result_i] = '\r';
		1103	result_i += 1;
		1104	},
		1105	'0' => {
		1106	i = backslash + "\\n".len;
		1107	buf[result_i] = 0;
		1108	result_i += 1;
		1109	},
		1110
		1111	'x' => {
		1112	const codepoint = std.fmt.parseInt(u8, esc[backslash + 2 ..][0..2], 16) catch
		1113	unreachable; // invalid string literal
		1114	i = backslash + "\\x00".len;
		1115	if (codepoint < 0x80) {
		1116	buf[result_i] = codepoint;
		1117	result_i += 1;
		1118	} else {
		1119	const len = std.unicode.wtf8Encode(codepoint, buf[result_i..][0..2]) catch
		1120	unreachable; // all codepoints <= 0xFF are valid
		1121	assert(len == 2);
		1122	result_i += 2;
		1123	}
		1124	},
		1125	// unicode literals contain a numerical parse
		1126	'u' => {
		1127	assert(esc[backslash + 2] == '{');
		1128	// skip 0x hex prefix if present
		1129	const start = if (esc[backslash + 4] == 'x')
		1130	backslash + 5
		1131	else
		1132	backslash + 3;
		1133	const end = mem.indexOfScalarPos(u8, esc, start, '}') orelse
		1134	unreachable; // invalid escape, missing }
		1135	const slice = esc[start..end];
		1136	const codepoint = std.fmt.parseInt(u21, slice, 16) catch
		1137	unreachable; // invalid numeric string
		1138	const len = std.unicode.utf8CodepointSequenceLength(codepoint) catch
		1139	unreachable; // invalid codepoint
		1140	const actual_len =
		1141	std.unicode.wtf8Encode(codepoint, buf[result_i..][0..len]) catch
		1142	unreachable; // already known to be valid codepoint
		1143	assert(len == actual_len);
		1144	result_i += len;
		1145	i = end + 1;
		1146	},
		1147
		1148	else => unreachable, // invalid escape
		1149	}
		1150	}
		1151	@memcpy(buf[result_i..][0 .. esc.len - i], esc[i..]);
		1152	return buf[0 .. result_i + esc.len - i];
		1153	}
		1154
		1155	/// Given a valid string literal contents (excluding the quotes),
		1156	/// decode it into a newly allocated slice.
		1157	pub fn decodeStringAlloc(alloc: mem.Allocator, esc: []const u8) ![]u8 {
		1158	const count = decodeStringCount(esc);
		1159	const memory = try alloc.alloc(u8, count);
		1160	const result = decodeStringBuf(esc, memory);
		1161	assert(result.ptr == memory.ptr);
		1162	assert(result.len == memory.len);
		1163	return memory;
		1164	}
		1165
988	const Tokenizer = @This();	1166	const Tokenizer = @This();
989	const std = @import("std");	1167	const std = @import("std");
990	const unicode = std.unicode;	1168	const unicode = std.unicode;
...	@@ -993,3 +1171,57 @@ const mem = std.mem;	...	@@ -993,3 +1171,57 @@ const mem = std.mem;
993	const unicode_props = @import("unicode_props.zig");	1171	const unicode_props = @import("unicode_props.zig");
994	const T = @import("tracer");	1172	const T = @import("tracer");
995	const Location = @import("Ast.zig").Location;	1173	const Location = @import("Ast.zig").Location;
		1174
		1175	fn testStringDecode(expected: []const u8, escaped: []const u8) !void {
		1176	const testing = std.testing;
		1177	const decoded = try decodeStringAlloc(testing.allocator, escaped);
		1178	defer testing.allocator.free(decoded);
		1179	try testing.expectEqualSlices(u8, expected, decoded);
		1180	}
		1181
		1182	test "basic tokenizer example" {
		1183	const testing = std.testing;
		1184	try unicode_props.init();
		1185
		1186	// not valid source code, but testing the tokenization
		1187	const source =
		1188	\\var token = 1.2e9.2 4;
		1189	\\// comment ignored
		1190	\\pub fn hello(:+ true) id return
		1191	\\π @"hello" "hello \r\n \x1b"
		1192	;
		1193	const expected_tokens = [_]Token{
		1194	.{ .tag = .kw_var, .offset = 0 },
		1195	.{ .tag = .id, .offset = 4 },
		1196	.{ .tag = .equal, .offset = 10 },
		1197	.{ .tag = .number_float, .offset = 12 },
		1198	.{ .tag = .number_int, .offset = 20 },
		1199	.{ .tag = .semi, .offset = 21 },
		1200	.{ .tag = .kw_pub, .offset = 42 },
		1201	.{ .tag = .kw_fn, .offset = 46 },
		1202	.{ .tag = .id, .offset = 49 },
		1203	.{ .tag = .start_paren, .offset = 54 },
		1204	.{ .tag = .colon, .offset = 55 },
		1205	.{ .tag = .plus, .offset = 56 },
		1206	.{ .tag = .id, .offset = 58 },
		1207	.{ .tag = .end_paren, .offset = 62 },
		1208	.{ .tag = .id, .offset = 64 },
		1209	.{ .tag = .kw_return, .offset = 67 },
		1210	.{ .tag = .id, .offset = 74 },
		1211	.{ .tag = .id, .offset = 77 },
		1212	.{ .tag = .string, .offset = 86 },
		1213	};
		1214	var tokenizer = Tokenizer{ .code = source };
		1215	for (expected_tokens) \|tok\| {
		1216	tokenizer.skipWhitespace();
		1217	try testing.expectEqual(tok, (try tokenizer.next()).?);
		1218	}
		1219	tokenizer.skipWhitespace();
		1220	try testing.expectEqual(null, (try tokenizer.next()));
		1221	}
		1222
		1223	test "string escapes" {
		1224	try testStringDecode("hello world", "hello world");
		1225	try testStringDecode("hello\n\rw\x00or\x5ald", "hello\\n\\rw\\0or\\x5ald");
		1226	try testStringDecode("255: \xC3\xBF", "255: \\xFF");
		1227	}

src/core/Transform.zig+18-1

...	@@ -707,7 +707,7 @@ fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) Alloc	...	@@ -707,7 +707,7 @@ fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) Alloc
707		707
708	.string => {	708	.string => {
709	const str = t.ast.basicNodeSlice(node);	709	const str = t.ast.basicNodeSlice(node);
710	const sliced = try t.addSlicedString(str[1 .. str.len - 1]);	710	const sliced = try t.addDecodedSlicedString(str[1 .. str.len - 1]);
711	return try t.addInstRef(.{	711	return try t.addInstRef(.{
712	.tag = .string,	712	.tag = .string,
713	.data = EncodedData.encode(.string, .{	713	.data = EncodedData.encode(.string, .{
...	@@ -1507,6 +1507,23 @@ fn addSlicedString(t: *Transform, str: []const u8) !SlicedString {	...	@@ -1507,6 +1507,23 @@ fn addSlicedString(t: *Transform, str: []const u8) !SlicedString {
1507	return string;	1507	return string;
1508	}	1508	}
1509		1509
		1510	fn addDecodedSlicedString(t: *Transform, escaped: []const u8) !SlicedString {
		1511	const decoded = if (Tokenizer.stringHasEscapes(escaped)) brk: {
		1512	const len = Tokenizer.decodeStringCount(escaped);
		1513
		1514	// use the scratch arraylist as a temporary storage for
		1515	// decoding the string. this is done so that duplicate
		1516	// strings will get de-duplicated in string_bytes
		1517	//
		1518	// TODO: see if this is a useless optimization
		1519	try t.scratch.ensureUnusedCapacity(t.gpa, @divFloor(len, @sizeOf(u32)) + 1);
		1520	const buf = mem.sliceAsBytes(t.scratch.unusedCapacitySlice());
		1521
		1522	break :brk Tokenizer.decodeStringBuf(escaped, buf);
		1523	} else escaped;
		1524	return t.addSlicedString(decoded);
		1525	}
		1526
1510	const Transform = @This();	1527	const Transform = @This();
1511	const std = @import("std");	1528	const std = @import("std");
1512	const mem = std.mem;	1529	const mem = std.mem;

src/core/array.zig+1-1

...	@@ -271,7 +271,7 @@ pub fn ExtraStore(comptime set_kind: StoreKind, comptime set_mutability: Mutabil	...	@@ -271,7 +271,7 @@ pub fn ExtraStore(comptime set_kind: StoreKind, comptime set_mutability: Mutabil
271		271
272	pub fn writeU32(w: *TrailWriter, value: u32) void {	272	pub fn writeU32(w: *TrailWriter, value: u32) void {
273	assert(w.leftover > 0);	273	assert(w.leftover > 0);
274	w.ip.extra.appendAssumeCapacity(value);	274	w.store.list.appendAssumeCapacity(value);
275	w.leftover -= 1;	275	w.leftover -= 1;
276	}	276	}
277		277

src/core/main.zig+1-1

...	@@ -22,7 +22,7 @@ pub fn main() !void {	...	@@ -22,7 +22,7 @@ pub fn main() !void {
22		22
23	try unicode_props.init();	23	try unicode_props.init();
24		24
25	if (builtin.mode == .Debug) {	25	if (builtin.mode == .Debug and std.io.getStdErr().isTty()) {
26	// zig build system doesnt print a newline :/	26	// zig build system doesnt print a newline :/
27	std.debug.print("\r\x1b[K", .{});	27	std.debug.print("\r\x1b[K", .{});
28	}	28	}

src/core/mod.zig+4

...	@@ -12,3 +12,7 @@ pub const Tokenizer = @import("Tokenizer.zig");	...	@@ -12,3 +12,7 @@ pub const Tokenizer = @import("Tokenizer.zig");
12	pub const Transform = @import("Transform.zig");	12	pub const Transform = @import("Transform.zig");
13	pub const unicode_props = @import("unicode_props.zig");	13	pub const unicode_props = @import("unicode_props.zig");
14	pub const util = @import("util.zig");	14	pub const util = @import("util.zig");
		15
		16	comptime {
		17	@import("std").testing.refAllDecls(@This());
		18	}

src/langref/generate-langref.zig created+358

...	@@ -0,0 +1,358 @@
1	//! Language reference compiler. Converts markdown into HTML, using
2	//! the Tokenizer to highlight code, and the compiler and node.js to
3	//! show code results. Only ever built in Debug mode. We are not
4	//! smart about memory. Written at 3am on a Thursday. Enjoy
5	pub const std = @import("std");
6	pub const core = @import("core");
7
8	const ArrayList = std.ArrayList;
9
10	pub fn main() !void {
11	comptime std.debug.assert(@import("builtin").mode == .Debug);
12	const gpa = std.heap.c_allocator;
13
14	var args = try std.process.argsWithAllocator(gpa);
15	_ = args.next().?;
16
17	const compiler_path = args.next().?;
18	const markdown_src = try std.fs.cwd().readFileAlloc(gpa, args.next().?, std.math.maxInt(usize));
19	const template_src = try std.fs.cwd().readFileAlloc(gpa, args.next().?, std.math.maxInt(usize));
20	const out_path = args.next().?;
21
22	var output = std.ArrayList(u8).init(gpa);
23	defer output.deinit();
24	try processMarkdown(
25	gpa,
26	markdown_src,
27	compiler_path,
28	output.writer(),
29	);
30
31	const applied_template = try std.mem.replaceOwned(
32	u8,
33	gpa,
34	template_src,
35	"{contents}",
36	output.items,
37	);
38	defer gpa.free(applied_template);
39
40	const out_file = try std.fs.createFileAbsolute(out_path, .{});
41	defer out_file.close();
42	try out_file.writeAll(applied_template);
43	}
44
45	const BlockType = enum {
46	para,
47	h1,
48	h2,
49	h3,
50	code_run,
51	code_build,
52	code,
53	code_generic,
54	ul,
55	table,
56	blockquote,
57	};
58
59	const InlineType = enum {
60	bold,
61	code,
62	};
63
64	const MarkdownTokenizer = struct {
65	/// Remaining text
66	str: []const u8,
67
68	fn skipWhitespace(t: *MarkdownTokenizer) void {
69	t.str = std.mem.trimLeft(u8, t.str, " \r\n");
70	}
71
72	fn advance(t: *MarkdownTokenizer, n: usize) []const u8 {
73	defer {
74	t.str = t.str[n..];
75	}
76	return t.str[0..n];
77	}
78
79	fn advanceLine(t: *MarkdownTokenizer) []const u8 {
80	return std.mem.trim(u8, t.advanceLineRaw(), " \r\n");
81	}
82
83	fn advanceLineRaw(t: *MarkdownTokenizer) []const u8 {
84	const n = std.mem.indexOfScalar(u8, t.str, '\n');
85	const line = t.advance(if (n) \|x\| x + 1 else t.str.len);
86	return line;
87	}
88
89	fn blockType(t: *MarkdownTokenizer) BlockType {
90	switch (t.str[0]) {
91	'#' => {
92	defer {
93	while (t.str[0] == '#')
94	_ = t.advance(1);
95	}
96
97	if (t.str[1] != '#') return .h1;
98	if (t.str[2] != '#') return .h2;
99	if (t.str[3] != '#') return .h3;
100	@panic("headers nested too deep");
101	},
102	'>' => {
103	_ = t.advance(1);
104	return .blockquote;
105	},
106	'\|' => {
107	return .table;
108	},
109	'`' => {
110	if (std.mem.startsWith(u8, t.str, "```")) {
111	defer _ = t.advanceLine();
112	if (std.mem.startsWith(u8, t.str, "```lang:run"))
113	return .code_run;
114	if (std.mem.startsWith(u8, t.str, "```lang:build"))
115	return .code_build;
116	if (std.mem.startsWith(u8, t.str, "```lang"))
117	return .code;
118	return .code_generic;
119	}
120	return .para;
121	},
122	'-' => {
123	return .ul;
124	},
125	else => {
126	return .para;
127	},
128	}
129	}
130	};
131
132	fn processMarkdown(gpa: std.mem.Allocator, src: []const u8, compiler_path: []const u8, writer: anytype) !void {
133	var t = MarkdownTokenizer{ .str = src };
134
135	t.skipWhitespace();
136	while (t.str.len > 0) : (t.skipWhitespace()) {
137	switch (t.blockType()) {
138	.para,
139	=> {
140	try writer.writeAll("<p>\n");
141
142	var para = ArrayList(u8).init(gpa);
143	try para.appendSlice(t.advanceLine());
144
145	var next_line = t.advanceLine();
146	while (t.str.len > 0 and next_line.len > 0) {
147	try para.append(' ');
148	try para.appendSlice(next_line);
149	next_line = t.advanceLine();
150	}
151
152	try processInlineMarkdown(gpa, para.items, writer);
153
154	try writer.writeAll("\n</p>\n");
155	},
156	.h1,
157	.h2,
158	.h3,
159	=> \|tag\| {
160	const html_tag = @tagName(tag);
161	const name = t.advanceLine();
162	try writer.print("<{0s}>{1s}</{0s}>\n", .{
163	html_tag,
164	name,
165	});
166	},
167	.code_run,
168	.code_build,
169	.code,
170	.code_generic,
171	=> \|tag\| {
172	var code = ArrayList(u8).init(gpa);
173
174	var next_line = t.advanceLineRaw();
175	while (t.str.len > 0 and !std.mem.startsWith(u8, next_line, "```")) {
176	try code.appendSlice(next_line);
177	next_line = t.advanceLineRaw();
178	}
179
180	try writer.writeAll("<pre><code>");
181	try writeTokenizedCode(gpa, code.items, writer);
182	try writer.writeAll("</code></pre>");
183
184	switch (tag) {
185	.code_run,
186	.code_build,
187	=> {
188	var build = std.process.Child.init(&.{ compiler_path, "/dev/stdin" }, gpa);
189	build.stdin_behavior = .Pipe;
190	build.stdout_behavior = .Pipe;
191	build.stderr_behavior = .Pipe;
192	var env_map = try std.process.getEnvMap(gpa);
193	defer env_map.deinit();
194	try env_map.put("YES_COLOR", "1");
195	build.env_map = &env_map;
196	try build.spawn();
197	try build.stdin.?.writeAll(code.items);
198	var out = ArrayList(u8).init(gpa);
199	var err = ArrayList(u8).init(gpa);
200	try build.collectOutput(&out, &err, std.math.maxInt(usize));
201	const result = try build.wait();
202	_ = result;
203	if (std.mem.trim(u8, err.items, " \n\r").len > 0) {
204	try writer.writeAll("<pre class='term'><code>");
205	try writeAnsi(gpa, err.items, writer);
206	try writer.writeAll("</code></pre>");
207	}
208	if (std.mem.trim(u8, out.items, " \n\r").len > 0) {
209	try writer.writeAll("<pre><code>");
210	try writer.writeAll(out.items);
211	try writer.writeAll("</code></pre>");
212	}
213	},
214	else => {},
215	}
216	},
217	.ul,
218	=> {
219	try writer.writeAll("<ul>");
220	while (t.str.len > 0 and (t.str[0] == '-')) {
221	_ = t.advance(1);
222	const line = t.advanceLine();
223	try writer.writeAll("<li>");
224	try processInlineMarkdown(gpa, line, writer);
225	try writer.writeAll("</li>");
226	}
227	try writer.writeAll("</ul>");
228	},
229	.table,
230	=> {
231	_ = t.advanceLine();
232	},
233	.blockquote,
234	=> {
235	try writer.writeAll("<blockquote>\n");
236
237	var para = ArrayList(u8).init(gpa);
238	try para.appendSlice(t.advanceLine());
239
240	var next_line = t.advanceLine();
241	while (t.str.len > 0 and next_line.len > 0) {
242	try para.append(' ');
243	try para.appendSlice(next_line);
244	next_line = t.advanceLine();
245	}
246
247	try processInlineMarkdown(gpa, para.items, writer);
248
249	try writer.writeAll("\n</blockquote>\n");
250	},
251	}
252	}
253	}
254
255	pub fn nextInlineToken(src: []const u8) ?struct { InlineType, usize } {
256	var pos: usize = 0;
257	while (true) : (pos += 1) {
258	pos = std.mem.indexOfAnyPos(u8, src, pos, "*`") orelse return null;
259
260	switch (src[pos]) {
261	'*' => switch (src[pos + 1]) {
262	'*' => return .{ .bold, pos },
263	else => continue,
264	},
265	'`' => return .{ .code, pos },
266	else => unreachable,
267	}
268	}
269	}
270
271	fn processInlineMarkdown(gpa: std.mem.Allocator, src: []const u8, writer: anytype) !void {
272	var bold = false;
273	var remain = src;
274
275	while (nextInlineToken(remain)) \|tok\| {
276	try writer.writeAll(remain[0..tok[1]]);
277	remain = remain[tok[1]..];
278	switch (tok[0]) {
279	.bold => {
280	bold = !bold;
281	try writer.writeAll(switch (bold) {
282	true => "<strong>",
283	false => "</strong>",
284	});
285	remain = remain[2..];
286	},
287	.code => {
288	const end = std.mem.indexOfScalarPos(u8, remain, 1, '`') orelse @panic("invalid code block");
289	const code = remain[1..end];
290	try writer.writeAll("<code>");
291	try writeTokenizedCode(gpa, code, writer);
292	try writer.writeAll("</code>");
293	remain = remain[end + 1 ..];
294	},
295	}
296	}
297	try writer.writeAll(remain);
298
299	if (bold) {
300	std.debug.panic("Unterminated Bold in '{s}'", .{src});
301	}
302	}
303
304	fn writeTokenizedCode(gpa: std.mem.Allocator, code: []const u8, writer: anytype) !void {
305	const code_z = try gpa.dupeZ(u8, code);
306	defer gpa.free(code_z);
307
308	var t = core.Tokenizer{ .code = code_z };
309
310	t.skipWhitespace();
311	var last: usize = 0;
312	while (true) {
313	t.skipWhitespace();
314	const tok = t.next() catch {
315	t.pos += 1;
316	continue;
317	} orelse break;
318	try writer.writeAll(t.code[last..tok.offset]);
319	try writer.print("<span class='tok-{s}{s}'>{s}</span>", .{
320	@tagName(tok.tag),
321	if (tok.tag.isKeyword()) " kw" else "",
322	t.code[tok.offset..t.pos],
323	});
324	last = t.pos;
325	}
326	}
327
328	fn writeAnsi(gpa: std.mem.Allocator, code: []const u8, writer: anytype) !void {
329	var remain = code;
330	var count: usize = 0;
331	while (std.mem.indexOfScalar(u8, remain, '\x1b')) \|esc_i\| {
332	try writer.writeAll(remain[0..esc_i]);
333	switch (remain[esc_i + 1]) {
334	'[' => switch (remain[esc_i + 2]) {
335	'0'...'9' => {
336	const end = std.mem.indexOfScalarPos(u8, remain, esc_i + 2, 'm').?;
337	const int = std.fmt.parseInt(u8, remain[esc_i + 2 .. end], 10) catch unreachable;
338	switch (int) {
339	else => {
340	try writer.print("<span class='ansi-{d}'>", .{int});
341	count += 1;
342	},
343	0 => {
344	for (0..count) \|_\| {
345	try writer.writeAll("</span>");
346	}
347	count = 0;
348	},
349	}
350	remain = remain[end + 1 ..];
351	},
352	else => \|x\| std.debug.panic("Unknown esc + 2: {d}", .{x}),
353	},
354	else => \|x\| std.debug.panic("Unknown esc + 1: {d}", .{x}),
355	}
356	}
357	_ = gpa;
358	}

src/langref/langref.template.html created+78

...	@@ -0,0 +1,78 @@
1	<!doctype html>
2	<html lang="en">
3	<head>
4	<meta charset="UTF-8">
5	<meta name="viewport" content="width=device-width, initial-scale=1.0">
6	<title>Language Reference</title>
7	<style>
8	* { margin: 0; padding: 0; box-sizing: border-box; }
9	body {
10	background-color: #fefefe;
11	font-family: sans-serif;
12	max-width: 45rem;
13	padding: 1rem;
14	margin: auto;
15	}
16	h1,h2,h3,h4,h5,h6,p,ul,pre,blockquote {
17	margin-bottom: 1rem;
18	}
19	ul {
20	padding-left: 2rem;
21	}
22	li {
23	margin-bottom: 0.25rem;
24	}
25	blockquote {
26	background-color: #ddd;
27	border-left: 4px solid black;
28	padding-left: 16px;
29	padding-top: 8px;
30	padding-bottom: 8px;
31	}
32	pre {
33	padding: 0.5rem;
34	}
35	pre > code {
36	border: none;
37	padding: 0;
38	background-color: transparent;
39	}
40	code, pre {
41	border: 1px solid #999;
42	background-color: #eeeeee;
43	border-radius: 4px;
44	padding: 2px 4px;
45	}
46	.term {
47	border: 1px solid #333;
48	background-color: #222;
49	color:white;
50	}
51	.ansi-1 {
52	font-weight: bold;
53	}
54	.ansi-31 {
55	color:rgb(255, 68, 68);
56	}
57	.ansi-2 {
58	color:#aaa;
59	}
60
61	.kw {
62	color: mediumpurple;
63	font-weight: bold;
64	}
65	.tok-string {
66	color: green;
67	}
68	.tok-number_int, .tok-number_float {
69	color: orangered;
70	}
71	</style>
72	</head>
73	<body>
74	<main>
75	{contents}
76	</main>
77	</body>
78	</html>

author	dave caruso <me@paperdave.net> 2024-05-31 04:56:12-07:00
committer	dave caruso <me@paperdave.net> 2024-05-31 04:56:12-07:00
log	1822706c309b53326056cd1030df93696fe321c8
tree	a645b076118013e76b36af50d5f21f2a6d393818
parent	eefe35dc33f3f2354626c64f066bb20d3f595496
signature	Commit is signed but in an unrecognized format.

add language reference