From 5f480e3bc33c572ac26aefc905d9426cad259954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Voisin?= Date: Sat, 21 Sep 2024 09:21:48 +0200 Subject: [PATCH] feat: new language manual design --- CHANGELOG.md | 1 + README.md | 29 +- build.zig | 39 +- ci | 15 +- ci.cmd | 14 + doc/language-manual.md | 255 ++++-- dprint.jsonc | 1 + src/core/Comp.zig | 2 +- src/core/InternPool.zig | 2 +- src/core/Sema.zig | 40 +- src/core/Teg.zig | 2 +- src/core/js/Link.zig | 12 +- src/core/mod.zig | 2 +- src/main.zig | 2 +- src/manual/NaiveHtmlMinifier.zig | 231 +++++ .../{generate-man.zig => generate_manual.zig} | 232 +++-- src/manual/template.html | 819 +++++++++++++++--- 17 files changed, 1385 insertions(+), 313 deletions(-) create mode 100644 ci.cmd create mode 100644 src/manual/NaiveHtmlMinifier.zig rename src/manual/{generate-man.zig => generate_manual.zig} (75%) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd71745e131b8617bc823abc843657391a016c7a..0e55238195a81a5e3ec6237f0e259933850c70fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## 0.1.7 +- chore: rename the language to "ix" - feat: allow any expression as a function body - fix: coerce `break` operands and implicit breaks to result type - fix: coerce results when `if` expression branches merge diff --git a/README.md b/README.md index 34342911d673b11ae23120ee79629e5ff1a9001b..c36e87250efe7a2648ecb5858ea9c7cdd919f603 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,34 @@ -# New Language Project +# Ix -This is a high-level, general purpose programming language. It is a mix of Rust, -Zig, and JavaScript's great points. +Ix (**I**nferable E**x**pression) is a high-level, general purpose programming +language. It features a mix of Rust, Zig, and JavaScript's great points. -For a guide on how to use the language, please see the Language Manual included -in the binary distribution. (TODO: host a rendered version) +For a guide on how to use Ix, see the manual included in the binary +distribution. Additionally, the latest commit of the manual is hosted online at +[language-manual.pages.dev](https://language-manual.pages.dev). ## Building -The primary dependency of the language SDK is a Zig compiler. If one is not +The primary dependency of the Ix compiler is a Zig compiler. If one is not available, the `./build` and `.\build.cmd` scripts can substitute for running `zig build` on POSIX and Windows systems, respectively. With a Zig compiler, a release build can be produced in a single command: zig build --release -To run the standard packaging procedure, which builds the SDK for all officially -supported targets: +To run the standard packaging procedure, which builds the compiler for all +officially supported targets: zig build package + +## Development + +Continuous integration is provided via a `./ci` (script for POSIX), which is run +by [self hosted Buildkite runners][ci]. This script uses the [`typos`][typos] +CLI to check spelling mistakes and [`dprint`][dprint] for code formatting. It is +recommended to setup editor integration with these two tools to prevent pushing +broken commits. + +[ci]: https://buildkite.com/paperdave/compiler +[typos]: https://github.com/crate-ci/typos +[dprint]: https://dprint.dev diff --git a/build.zig b/build.zig index 18e0ca831f79de13881cd284060a88d42a0c1385..58f4952d58492c27162cbf6fdaa99c4d90a2c0c2 100644 --- a/build.zig +++ b/build.zig @@ -18,8 +18,8 @@ pub const Options = struct { config: BuildConfig, }; -/// `createArtifactGroup` multiple times in order to have different -/// targets and configurations of the SDK at the same time. +/// `createArtifactGroup` is called multiple times in order to have different +/// targets and configurations of Ix at the same time. const ArtifactGroup = struct { build: *Build, opts: Options, @@ -446,10 +446,10 @@ pub fn build(b: *Build) !void { test_all.dependOn(run_step); } - // Language Reference Generator + // Language Manual Generator const manual_generator = b.addExecutable(.{ - .name = "generate-man", - .root_source_file = b.path("src/manual/generate-man.zig"), + .name = "generate_manual", + .root_source_file = b.path("src/manual/generate_manual.zig"), .target = b.graph.host, .optimize = .Debug, .link_libc = true, @@ -533,30 +533,21 @@ pub fn build(b: *Build) !void { .version = version, .have_embedded_quickjs = have_embedded_quickjs, }, - .optimize = optimize, + .optimize = .ReleaseSafe, .target = cross_target, .no_llvm = no_llvm, }); - inline for (.{ - .{ artifacts.cli, "compiler" }, - }) |entry| { - const artifact, const bin_name = entry; - package.dependOn(&b.addInstallFile( - artifact.getEmittedBin(), - b.pathJoin(&.{ - "package", - name, - b.fmt("{s}{s}", .{ - bin_name, - cross_target.result.exeFileExt(), - }), - }), - ).step); - } - + package.dependOn(&b.addInstallFile( + artifacts.cli.getEmittedBin(), + b.pathJoin(&.{ + "package", + name, + b.fmt("ix{s}", .{cross_target.result.exeFileExt()}), + }), + ).step); package.dependOn(&b.addInstallFile(b.path("CHANGELOG.md"), "package/CHANGELOG.md").step); - package.dependOn(&b.addInstallFile(manual_out, "package/README.html").step); + package.dependOn(&b.addInstallFile(manual_out, "package/ix.html").step); } { diff --git a/ci b/ci index 1b32e72911810d8cc7b55936344c096ac9a578a0..556ae5ae133efd4692e72c47cf8a677c64034e93 100755 --- a/ci +++ b/ci @@ -11,9 +11,11 @@ commit_short=$(echo "$commit" | cut -c -9) ./build --version typos dprint check -./build all --summary new if [ -n "$BUILDKITE" ]; then + # perform the entire build + ./build all --summary all + pushd zig-out/package zip -r compiler-$commit.zip * buildkite-agent artifact upload compiler-$commit.zip @@ -21,12 +23,15 @@ if [ -n "$BUILDKITE" ]; then rm -fr zig-out/site mkdir zig-out/site - cp zig-out/package/README.html zig-out/site/index.html - branch=$BUILDKITE_BRANCH + cp zig-out/package/ix.html zig-out/site/index.html + branch="$BUILDKITE_BRANCH" if [ "$branch" = "dev" ]; then branch=main fi - CLOUDFLARE_API_TOKEN=$(buildkite-agent secret get CLOUDFLARE_API_TOKEN) \ - CLOUDFLARE_ACCOUNT_ID=$(buildkite-agent secret get CLOUDFLARE_ACCOUNT_ID) \ + CLOUDFLARE_API_TOKEN="$(buildkite-agent secret get CLOUDFLARE_API_TOKEN)" \ + CLOUDFLARE_ACCOUNT_ID="$(buildkite-agent secret get CLOUDFLARE_ACCOUNT_ID)" \ wrangler pages deploy zig-out/site --branch "$branch" --commit-hash "$commit" --no-bundle --project-name language-manual +else + # perform only the tests, with a shorter summary + ./build test --summary new fi diff --git a/ci.cmd b/ci.cmd new file mode 100644 index 0000000000000000000000000000000000000000..4a48b012cc5bd1ffc68650d2225ed3badad64ceb --- /dev/null +++ b/ci.cmd @@ -0,0 +1,14 @@ +@ECHO OFF +@REM Continuous Integration Script +@REM This is a local-only version of ./ci as none of the +@REM self-hosted CI runners use Windows. This is just +@REM for local verification of builds. + +typos +if %errorlevel% neq 0 exit /b %errorlevel% + +dprint check +if %errorlevel% neq 0 exit /b %errorlevel% + +build.cmd test +if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/doc/language-manual.md b/doc/language-manual.md index c26d08865ca2495a2e4e3e0178a65436151f2382..ffca26d424414617bbe800242efbf1d5dc49b9b3 100644 --- a/doc/language-manual.md +++ b/doc/language-manual.md @@ -1,54 +1,65 @@ -# New Language Project +# Ix Programming Language -This is a high-level, general purpose programming language. +Ix (**I**nferable E**x**pression) is a high-level programming language for +developing web applications and games. Key goals of this project are: -- **Ease of Use**: Use of type inference keeps code concise and easy to read and - write. -- **Fast Iteration**: The compiler is designed for quick incremental compilation - and hot module reloading. (note: incremental is not yet available) -- TODO: some more +- **Ease of Use**: Use of type inference and simple syntax makes code easy to + read and write. +- **Fast Iteration**: Incremental compilation keeps development productivity + high. +- **Safety**: Compiler errors catch most mistakes at compile-time, and + additional tooling to catch mistakes at runtime. +- **Universal**: One codebase can seamlessly target different web browsers and + native targets. It primarily focuses on merging some great points of Rust, Zig, and JavaScript's language designs: -- **Rust** - Expressive control flow that is easy to write. -- **Zig** - Execute any code at compile time; everything is an expression. -- **JavaScript** - Garbage collected values with a versatile type system. +- [**Rust**](https://www.rust-lang.org) - Expressive and concise control flow + that is easy to write. +- [**Zig**](https://ziglang.org) - Execute any code at compile time; everything + is an expression. +- [**JavaScript**](https://developer.mozilla.org/en-US/docs/Web/JavaScript) - + Garbage collected values with a versatile type system. -## Standard Library - -Currently, there is no standard library. One will be introduced later. +This manual is a standalone HTML document. Code samples are compiled and tested +as a part of the release cycle. ## Hello World ```lang:run let print = @extern("console").log; - fn main { print("Hello, world!"); } ``` The `print` function is implemented by getting an external reference to the -`console` JavaScript global and then retrieving the `log` property from it. +`console` JavaScript global, retrieving the `log` property from it, and then +calling it when the program starts. + +> In the future, common APIs will be available in the standard library, such as +> `std.log.info()`. For now, no such library exists. ## Primitive Data Types -| name | ts equivalent | description | -| -------- | --------------------- | ---------------------------------------------- | -| `int` | `number` | a number in signed 32-bit range | -| `float` | `number` | a 64-bit floating point number | -| `bool` | `boolean` | either `true` or `false` | -| `string` | `string` | a sequence of text characters | -| `void` | `undefined` or `null` | detectable lack of a value | -| `opaque` | `unknown` or `any` | could be any value | -| `never` | `never` | control flow prevents this value from existing | +| name | ts equivalent | description | +| ---------- | ----------------------------- | ---------------------------------------------- | +| `bool` | `boolean` | either `true` or `false` | +| `float` | `number` | a 64-bit floating point number | +| `int` | `number` | an integer number in signed 32-bit range | +| `bigint` | `bigint` | an integer number with no range limit | +| `string` | `string` | a sequence of text characters | +| `void` | `undefined` / `null` / `void` | detectable lack of a value | +| `opaque` | `unknown` / `any` | could be any value | +| `noreturn` | `never` | control flow prevents this value from existing | -> **note**: `bigint` is planned but not added +> **Note**: Unlike JavaScript, the values `undefined` and `null` are merged into +> one concept: [void](#void) -## Built-in Constants +### Built-in Constants In addition to the above types being built-in constants, the following are available. @@ -62,9 +73,48 @@ available. These constants are reserved names, and it is an error to re-declare them. ```lang:build -let true = 1 == 0; +fn main { + let true = 1 == 0; +} ``` +### Numbers + +Numbers have the following rules: + +- May start with a `.`, but cannot end with `.` +- May contain leading zeros, but transform pass will warn. +- May contain a `_` separator, cannot have two in a row. +- May contain `e` or `E` for exponents. +- May use 0x, 0b, 0o prefixes for hex, binary, octal numbers. + +### Strings + +Numbers + +### Multi-line string literals + +```lang +let bwaa = + \\hello + \\ world + \\end +; +``` + +### Void + +Void is the lack a value. For example, a function that does nothing returns +`void`. + +```lang +pub fn hello() { + // do nothing +} +``` + + + ## Variables There are two types of variables, `let` and `var`. @@ -74,24 +124,30 @@ expression within. Referring to these identifiers is equivalent to inlining it where it was used, aside from side effect duplication. ```lang:run -let hello: int = 1; -let world = hello + 2; -let hello = hello * world; +let print = @extern("console").log; -@extern("console").log(hello); +fn main { + let hello: int = 1; + let world = hello + 2; + let hello = hello * world; + + print(hello); +} ``` `var` bindings create mutable memory locations at runtime. This means they can be re-assigned to: ```lang:run -var message = "first message"; +fn main { + var message = "first message"; -if @extern("Math").random() > 0.5 { - message = "second message"; + if @extern("Math").random() > 0.5 { + message = "second message"; + } + + @extern("console").log(message); } - -@extern("console").log(message); ``` Using `let` is preferred whenever possible, as it will lead to more readable @@ -99,13 +155,15 @@ code, and the compiler is able to optimize it better. The above can be rewritten as: ```lang:run -let message = if @extern("Math").random() > 0.5 { - "second message" -} else { - "first message" -}; +fn main { + let message = if @extern("Math").random() > 0.5 { + "second message" + } else { + "first message" + }; -@extern("console").log(message); + @extern("console").log(message); +} ``` It should be noted that numeric literals are a comptime-only data type, meaning @@ -114,7 +172,7 @@ it cannot be assigned to a `var` without an explicit type annotation. Since float, but a compiler warning will be emitted. ```lang -var number = 0; // `number` is a `float`, but not explicit +var number = 0; // `number` is a `float`, but not explicit var number: int = 0; // `number` is `int` var number: float = 0; // `number` is `float` ``` @@ -125,12 +183,15 @@ Functions are declared with the `fn` keyword. Parameter types must be annotated. The function body is a block, which may use implicit return to return. ```lang:run +let print = @extern("console").log; + fn add(a: int, b: int) -> int { a + b } -let print = @extern("console").log; -print("Sum of 2 and 7:", add(2, 7)); +fn main { + print("Sum of 2 and 7:", add(2, 7)); +} ``` > **Bug**: Return types are currently required. Omitting the `-> type` syntax @@ -140,6 +201,8 @@ print("Sum of 2 and 7:", add(2, 7)); The `return` expression can be used within functions to return early. ```lang:run +let print = @extern("console").log; + fn something(a: int) -> bool { if a == 0 { return false; @@ -148,8 +211,9 @@ fn something(a: int) -> bool { a > 100 } -let print = @extern("console").log; -print(something(0xFF), something(0)); +fn main { + print(something(0xFF), something(0)); +} ``` ## Number Literals @@ -178,7 +242,9 @@ fn currentYear() -> int { @jsNew(@extern("Date")).getFullYear() } -print(currentYear()); +fn main { + print(currentYear()); +} ``` A field access on an external handle is allowed to be performed at compile time, @@ -197,19 +263,21 @@ struct Position { y: int, } -let center = Position{ x: 0, y: -2 }; -print(center.x, center.y); +fn main { + let center = Position{ x: 0, y: -2 }; + print(center.x, center.y); -fn addPositions(a: Position, b: Position) -> Position { // To allow less verbose code, the struct type can be inferred with `.` + let added = addPositions( + .{ x: 1, y: -4 }, + .{ x: 5, y: 8 }, + ); + print("Add result:", added.x, added.y); +} + +fn addPositions(a: Position, b: Position) -> Position { .{ x: a.x + b.x, y: a.y + b.y } } - -let added = addPositions( - .{ x: 1, y: -4 }, - .{ x: 5, y: 8 }, -); -print("Add result:", added.x, added.y); ``` To allow optimizations, `struct`s by default do not have a defined @@ -217,11 +285,80 @@ representation in JavaScript. This means passing the struct to an opaque function is not allowed. ```lang:build +let print = @extern("console").log; + struct Person { name: string, age: int, } -let print = @extern("console").log; -print(Person{ name: "hello", age: 42 }); +fn main { + print(Person{ name: "hello", age: 42 }); +} ``` + +## A + +### B + +## C + +### a + +### somewhat long header name but still is within dave's reasonable-ish constraints + +#### Number Literals 2 + +TODO + +#### String Literals 2 + +TODO + +#### String WAdsafdsa + +TODO + +### hewwo + +cat + +### cat + +#### yeah + +## what + +## omg + +## Lorem ipsum dolor sit + +## amet, consectetur adipiscing + +## elit. Proin vulputate ligula + +## ac risus blandit, eu accumsan + +## rna feugiat. Phasellus quis + +## metus felis. Semper mattis + +## augue eu elementum. Sed nec odio + +## cursus neque fringilla rutrum. + +## Cras vestibulum bibendum hendrerit. + +## Duis porta, leo vel + +## sollicitudin rhoncus, arcu nulla + +## lobortis est, vitae interdum + +## quam ante eu ante. Integer luctus + +## justo a odio cursus auctor. Phasellus + +## blandit urna vitae quam dapibus + +## efficitur. Proin sagittis urna non diff --git a/dprint.jsonc b/dprint.jsonc index d5ecbae66371478101102d84ea719c1cf0d23f0e..a4f51e8c2bcde9c62c1998c87bbb5388fd97c48c 100644 --- a/dprint.jsonc +++ b/dprint.jsonc @@ -2,6 +2,7 @@ "plugins": [ // sort-lines: start "https://plugins.dprint.dev/exec-0.5.0.json@8d9972eee71fa1590e04873540421f3eda7674d0f1aae3d7c788615e7b7413d0", + "https://plugins.dprint.dev/g-plane/markup_fmt-v0.12.0.wasm", "https://plugins.dprint.dev/json-0.19.3.wasm", "https://plugins.dprint.dev/markdown-0.17.2.wasm", // sort-lines: end diff --git a/src/core/Comp.zig b/src/core/Comp.zig index bf511722ceea1e75aee9457c1f78a7f90a7d8e0c..118b19df45534443da4ddd7fd4b786b25d8d1e41 100644 --- a/src/core/Comp.zig +++ b/src/core/Comp.zig @@ -666,7 +666,7 @@ pub fn indexTegModule(comp: *Comp, file: *File, module_index: Teg.Inst.Index) !N const Options = struct { dump_ir: bool = false, - verbose_sema: bool = true, + verbose_sema: bool = false, }; /// The compiler backend is stored separately from the compilation, and diff --git a/src/core/InternPool.zig b/src/core/InternPool.zig index 9001b9a78d2fc4b917124b600606a8e32e1c72c8..01aea56e759f7bded40c5df6bb801c5c634ec0ca 100644 --- a/src/core/InternPool.zig +++ b/src/core/InternPool.zig @@ -2,7 +2,7 @@ //! https://github.com/ziglang/zig/blob/0.12.0/src/InternPool.zig //! //! All interned objects have both a Value and a Type. Type is a subset of -//! Value, since the language allows operating on types the same as values. +//! Value, since Ix allows operating on types the same as values. //! This data structure is intended to be serializable for incremental //! compilation efforts. //! diff --git a/src/core/Sema.zig b/src/core/Sema.zig index 0a8846387ce3325a24496a4ce1921abbf6894c12..1366aed00beae7d9b392bff8adddcefe7a381346 100644 --- a/src/core/Sema.zig +++ b/src/core/Sema.zig @@ -693,6 +693,27 @@ fn stepInner(s: *Sema, comptime tag: Teg.Tag) Error!InstructionKind.fromTag(tag) const lhs_type = s.typeOfRef(lhs); if (lhs_type == .@"opaque") { + if (lhs.value()) |value| { + if (value.valueTag(s.ip) == .external_value) { + const rhs = try s.getCoercedValue(s.lazyNode(.rhs), field.rhs, .string); + const field_key = rhs.asString(s.ip).tryToNullTerminatedString(s.ip, s.gpa) catch { + @panic("TODO: support null in these strings"); + }; + + const previous_key = value.toKey(s.ip).external_value.key; + + const new_key = try std.mem.concat(s.arena(), NullTerminatedString, &.{ + previous_key, + &.{field_key}, + }); + defer s.arena().free(new_key); + + return s.makeValueAir(.{ .external_value = .{ + .key = new_key, + } }); + } + } + try s.ensureAtRuntime(s.lazyNode(.lhs)); return s.addRef(.field_opaque, .{ .lhs = lhs, @@ -704,25 +725,6 @@ fn stepInner(s: *Sema, comptime tag: Teg.Tag) Error!InstructionKind.fromTag(tag) return try s.fieldAccessStruct(lhs, lhs_type, struct_type, field.rhs); } else if (lhs_type.toTupleType(s.ip)) |tuple_type| { return try s.fieldAccessTuple(lhs, lhs_type, tuple_type, field.rhs); - } else if (lhs.value()) |value| { - if (value.valueTag(s.ip) == .external_value) { - const rhs = try s.getCoercedValue(s.lazyNode(.rhs), field.rhs, .string); - const field_key = rhs.asString(s.ip).tryToNullTerminatedString(s.ip, s.gpa) catch { - @panic("TODO: support null in these strings"); - }; - - const previous_key = value.toKey(s.ip).external_value.key; - - const new_key = try std.mem.concat(s.arena(), NullTerminatedString, &.{ - previous_key, - &.{field_key}, - }); - defer s.arena().free(new_key); - - return s.makeValueAir(.{ .external_value = .{ - .key = new_key, - } }); - } } return s.fail(.field_invalid_type, .{ diff --git a/src/core/Teg.zig b/src/core/Teg.zig index d207cf72561b3ab027aa7e5859f565dc9b76dd4f..11b844392eb6864598b6b79f087ba3af50f1b51e 100644 --- a/src/core/Teg.zig +++ b/src/core/Teg.zig @@ -1,4 +1,4 @@ -//! TODO: rename this to something else. Teg is not a graph +//! TODO: rename this to EIR (expression intermediate representation) insts: Inst.List.Slice, string_bytes: []const u8, diff --git a/src/core/js/Link.zig b/src/core/js/Link.zig index a60404f119611e2fd5e5823044f36caebf92c403..fb436e07f7413b812bdb416542ebfb4944faa6b9 100644 --- a/src/core/js/Link.zig +++ b/src/core/js/Link.zig @@ -76,7 +76,7 @@ pub fn linkJsCode(link: *Link, entry_decl: Decl.Index, writer: Writer) !void { } try writer.writeAll(std.fmt.comptimePrint( - "// Language Compiler v{}. Alpha quality software.\n", + "// ix compiler v{}. alpha quality software.\n", .{@import("config").version}, )); @@ -514,16 +514,18 @@ fn printExprLevel( try writer.writeAll(")"); }, + .js_new => { const call: Tree.Call = tree.nodeData(.js_new, node); const args = call.args.get(tree); - if (Level.new.needsParens(level)) + const wrap = Level.new.needsParens(level) and level != .member; + if (wrap) try writer.writeAll("("); try writer.writeAll("new "); try link.printExpr(tree, call.base, writer); - if (args.len > 0) { + if (args.len > 0 or level == .member) { try writer.writeAll("("); for (args.slice(tree), 0..) |arg, i| { @@ -533,14 +535,14 @@ fn printExprLevel( try writer.writeAll(")"); } - if (Level.new.needsParens(level)) + if (wrap) try writer.writeAll(")"); }, .field => { const field: Tree.Field = tree.nodeData(.field, node); - try link.printExpr(tree, field.base, writer); + try link.printExprLevel(tree, field.base, .member, writer); try writer.writeAll("."); try writer.writeAll(field.field.toSlice(&link.comp.intern_pool)); }, diff --git a/src/core/mod.zig b/src/core/mod.zig index 23014e0f5a75b2f3310af971a8324b8790dc9afe..5329216e139b9bcb06515327656c5a6832e3401e 100644 --- a/src/core/mod.zig +++ b/src/core/mod.zig @@ -1,4 +1,4 @@ -//! Core language compiler module. +//! Core module for the Ix compiler // sort-lines: start diff --git a/src/main.zig b/src/main.zig index 2a2c61c5049c039497e9d8693d30c3871f2970cb..531b4dc0d1a694c3b30fd0b916c6a2ccd97b3144 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1,7 +1,7 @@ //! This is the user-facing command line, combining all features into //! different subcommands, each with their own option set. -const default_program_name = "wlang"; // fallback when argv[0] is missing +const default_program_name = "ix"; // fallback when argv[0] is missing const Command = struct { name: [:0]const u8, diff --git a/src/manual/NaiveHtmlMinifier.zig b/src/manual/NaiveHtmlMinifier.zig new file mode 100644 index 0000000000000000000000000000000000000000..61ed6588c769079ffee39acdbab95927534dbda3 --- /dev/null +++ b/src/manual/NaiveHtmlMinifier.zig @@ -0,0 +1,231 @@ +//! Removes whitespace between HTML/CSS markup. +//! This function is exclusively intended for `template.html`, +//! and doesn't handle many edge cases. +input: []const u8, +output: std.ArrayList(u8), + +const ws = " \t\r\n"; + +pub fn minify(input: []const u8, allocator: Allocator) ![]u8 { + var m: NaiveHtmlMinifier = .{ + .input = input, + .output = std.ArrayList(u8).init(allocator), + }; + errdefer m.output.deinit(); + + while (m.input.len > 0) { + m.skipWhitespace(); + try m.processHtml(); + } + + return try m.output.toOwnedSlice(); +} + +pub inline fn minifyComptime(input: []const u8) []const u8 { + comptime { + @setEvalBranchQuota(99999999); + var buf: [input.len * 2]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&buf); + const result_slice = minify(input, fba.allocator()) catch unreachable; // should not increase size + const result = result_slice[0..].*; + return &result; + } +} + +fn skipWhitespace(m: *NaiveHtmlMinifier) void { + var i: usize = 0; + while (m.input.len > i) switch (m.input[i]) { + ' ', '\t', '\n', '\r' => i += 1, + else => break, + }; + m.input = m.input[i..]; +} + +fn processHtml(m: *NaiveHtmlMinifier) !void { + switch (m.input[0]) { + ' ', '\t', '\n', '\r' => unreachable, // did not skip whitespace + + '<' => switch (m.input[1]) { + '!' => try m.moveTokenWith(">"), + '/' => try m.moveTokenWith(">"), + else => { + const first_tok = getToken(m, ws); + try m.moveToken(ws); + if (first_tok[first_tok.len - 1] == '>') { + try m.maybeProcessBody(first_tok[1 .. first_tok.len - 1]); + return; + } + const tag_name = first_tok[1..]; + var need_space_before_attr = true; + while (true) switch (m.input[0]) { + ' ', '\t', '\n', '\r' => unreachable, // did not skip whitespace + '>' => { + m.input = m.input[1..]; + try m.output.appendSlice(">"); + try m.maybeProcessBody(tag_name); + return; + }, + '/' => { + const tok = m.getToken(">"); + m.input = m.input[tok.len + 1 ..]; + try m.output.appendSlice("/>"); + try m.maybeProcessBody(tag_name); + return; + }, + else => { + if (need_space_before_attr) + try m.output.appendSlice(" "); + + try m.moveToken("=>" ++ ws); + if (m.input[0] == '=') { + m.input = m.input[1..]; + m.skipWhitespace(); + try m.output.appendSlice("="); + + // move attr value + const end = switch (m.input[0]) { + '"' => brk: { + need_space_before_attr = false; + try m.output.append('"'); + m.input = m.input[1..]; + break :brk "\""; + }, + '\'' => brk: { + need_space_before_attr = false; + try m.output.append('\''); + m.input = m.input[1..]; + break :brk "'"; + }, + else => brk: { + need_space_before_attr = true; + break :brk ws ++ ">"; + }, + }; + try m.moveTokenWith(end); + + m.skipWhitespace(); + } + }, + }; + }, + }, + + else => { + const text_block = m.getToken("<"); + m.input = m.input[text_block.len..]; + var it = mem.tokenizeAny(u8, text_block, ws); + try m.output.appendSlice(it.next().?); + while (it.next()) |token| { + try m.output.append(' '); + try m.output.appendSlice(token); + } + }, + } +} + +fn maybeProcessBody(m: *NaiveHtmlMinifier, tag_name: []const u8) !void { + inline for (.{ .style, .script, .pre }) |tag| { + if (mem.eql(u8, tag_name, @tagName(tag))) { + const end = mem.indexOf(u8, m.input, "") orelse @panic("no end"); + const tok = m.input[0 .. end + @tagName(tag).len + 3]; + + if (tag == .style) { + try m.processCss(m.input[0..end]); + } else { + try m.output.appendSlice(tok); + } + + m.input = m.input[end..]; + } + } +} + +fn processCss(m: *NaiveHtmlMinifier, input_css: []const u8) !void { + var without_comments = try std.ArrayList(u8).initCapacity(m.output.allocator, input_css.len); + defer without_comments.deinit(); + + { + var i: usize = 0; + while (mem.indexOfScalarPos(u8, input_css, i, '/')) |j| { + if (j + 1 < input_css.len and input_css[j + 1] == '*') { + try without_comments.appendSlice(input_css[i..j]); + i = mem.indexOfPos(u8, input_css, j + 2, "*/") orelse + std.debug.panic("no end comment in {s}", .{input_css[j..]}); + } else { + i = j + 1; + } + } + try without_comments.appendSlice(input_css[i..]); + } + + var cm: NaiveHtmlMinifier = .{ .input = without_comments.items, .output = undefined }; + cm.skipWhitespace(); + + while (cm.input.len > 0) { + const part = cm.getToken("{};"); + const last = cm.input[part.len]; + cm.input = cm.input[part.len + 1 ..]; + + switch (last) { + '{' => { + try appendWithoutCssComment(part, &m.output); + }, + ';', '}' => rule: { + if (part.len == 0) { + if (m.output.items[m.output.items.len - 1] == ';') + m.output.items.len -= 1; + break :rule; + } + + const i = mem.indexOfScalar(u8, part, ':') orelse + std.debug.panic("missing colon in {s}", .{part}); + const key = part[0..i]; + const value = part[i + 1 ..]; + + try appendWithoutCssComment(key, &m.output); + try m.output.append(':'); + try appendWithoutCssComment(value, &m.output); + }, + else => @panic("no end"), + } + + try m.output.append(last); + cm.skipWhitespace(); + } +} + +fn getToken(m: *const NaiveHtmlMinifier, ends: []const u8) []const u8 { + return m.input[0 .. mem.indexOfAny(u8, m.input, ends) orelse return m.input]; +} + +fn moveToken(m: *NaiveHtmlMinifier, ends: []const u8) !void { + const tok = m.getToken(ends); + m.input = m.input[tok.len..]; + m.skipWhitespace(); + try m.output.appendSlice(tok); +} + +fn moveTokenWith(m: *NaiveHtmlMinifier, ends: []const u8) !void { + const tok = m.getToken(ends); + try m.output.appendSlice(m.input[0 .. tok.len + 1]); + m.input = m.input[tok.len + 1 ..]; + m.skipWhitespace(); +} + +fn appendWithoutCssComment(text: []const u8, output: *std.ArrayList(u8)) !void { + var it = mem.tokenizeAny(u8, text, ws); + var needs_space = false; + while (it.next()) |tok| { + if (needs_space) try output.append(' '); + try output.appendSlice(tok); + needs_space = switch (tok[tok.len - 1]) { + ',' => false, + else => true, + }; + } +} + +const NaiveHtmlMinifier = @This(); +const std = @import("std"); +const mem = std.mem; +const Allocator = mem.Allocator; diff --git a/src/manual/generate-man.zig b/src/manual/generate_manual.zig similarity index 75% rename from src/manual/generate-man.zig rename to src/manual/generate_manual.zig index 81ae60d2318b5dcebdb1b7ed708b8f71d28b1e0e..6f4ff77e5d66bbcf9dc87cc29a5bc1b2992f51c3 100644 --- a/src/manual/generate-man.zig +++ b/src/manual/generate_manual.zig @@ -1,9 +1,10 @@ //! Language reference compiler. Converts markdown into HTML, using -//! the Tokenizer to highlight code, and the compiler and node.js to +//! the Tokenizer to highlight code, and the compiler and QJS to //! show code results. Only ever built in Debug mode. We are not //! smart about memory. Initially written at 3am on a Thursday. Enjoy pub const std = @import("std"); pub const core = @import("core"); +const NaiveHtmlMinifier = @import("NaiveHtmlMinifier.zig"); const mem = std.mem; const Allocator = mem.Allocator; @@ -12,13 +13,17 @@ const ArrayListUnmanaged = std.ArrayListUnmanaged; const assert = std.debug.assert; comptime { - assert(core.runtime_safety); + assert(@import("builtin").mode == .Debug); } +const cli_name = "wlang"; +const build_cli = cli_name ++ " build"; +const test_cli = cli_name ++ " test"; +const run_cli = cli_name ++ " run"; + pub fn main() !void { comptime std.debug.assert(@import("builtin").mode == .Debug); - var gpa_state: std.heap.GeneralPurposeAllocator(.{}) = .{}; - const gpa = gpa_state.allocator(); + const gpa = std.heap.c_allocator; const progress = std.Progress.start(.{ .root_name = "Build manual", @@ -39,6 +44,8 @@ pub fn main() !void { var contents = ArrayList(u8).init(gpa); var header: HeaderWriter = .{}; + const minify = false; + try processMarkdown( gpa, progress, @@ -51,10 +58,15 @@ pub fn main() !void { try header.end(gpa); - const applied_template = try replaceInTemplate(gpa, template_src, .{ + const minified_template = if (minify) + try NaiveHtmlMinifier.minify(template_src, gpa) + else + template_src; + + const applied_template = try replaceInTemplate(gpa, minified_template, .{ .contents = contents.items, .navigation = header.data.items, - .nav_h1 = header.h1.?, + .nav_h1 = "Ix Language", .repository = "https://github.com/paperdave/wl", }); @@ -169,27 +181,45 @@ const HeaderWriter = struct { level: u8 = 0, h1: ?[]const u8 = null, - pub fn add(header: *HeaderWriter, alloc: Allocator, name: []const u8, id: []const u8, level: u8) !void { + numbers: [5]u32 = .{ 0, 0, 0, 0, 0 }, + + has_closed_li: bool = true, + + fn add(header: *HeaderWriter, alloc: Allocator, name: []const u8, id: []const u8, level: u8) !void { const w = header.data.writer(alloc); + if (header.level == level) { + try w.writeAll(""); + } + while (header.level > level) { header.level -= 1; - try w.writeAll(""); + header.numbers[header.level] = 0; + try w.writeAll(""); } while (header.level < level) { if (header.level == 0) { try w.writeAll("