1const languages = [
2 "astro",
3 "css",
4 "diff",
5 "dosbatch",
6 "html",
7 "js",
8 "json",
9 "jsx",
10 "lua",
11 "mdx",
12 "php",
13 "powershell",
14 "python",
15 "shell",
16 "toml",
17 "ts",
18 "tsx",
19 "xml",
20 "yaml",
21 "zig",
22] as const;
23const altScopes: Record<string, string> = {
24 html: "text.html.basic",
25 astro: "text.html.astro",
26 xml: "text.xml",
27 php: "text.html.php",
28};
29export type Language = (typeof languages)[number];
30
31const scopes = [
32 // HTML
33 ["meta.tag.metadata.doctype.html", "builtin"],
34 ["meta.tag.structure", "keyword"],
35 ["meta.tag.metadata.script", "keyword"],
36
37 // CSS
38 ["punctuation.definition.keyword", "keyword", "css"],
39 ["entity.name.tag.css", "class", "css"],
40 ["meta.selector.css", "method", "css"],
41 ["entity.other.attribute-name.class.css", "builtin", "css"],
42 ["punctuation.definition.entity", "builtin", "css"],
43 ["variable.css", "parameter", "css"],
44
45 // JSON
46 ["support.type.property-name.json", "variable", "json"],
47 ["constant.numeric", "method", "json"],
48 ["constant", "class", "json"],
49
50 // Lua
51 ["entity.name", "class", "lua"],
52
53 // Diff
54 ["punctuation.definition.deleted", "variable", "diff"],
55 ["markup.deleted", "variable", "diff"],
56 ["punctuation.definition.inserted", "method", "diff"],
57 ["markup.inserted", "method", "diff"],
58 ["meta.diff.range", "string", "diff"],
59 ["punctuation.definition.range", "string", "diff"],
60 ["meta.toc-list-line.number", "keyword", "diff"],
61 ["meta.diff", "comment", "diff"],
62
63 // General
64 ["meta.object-literal.key", "property"],
65 ["comment", "comment"],
66 ["string", "string"],
67 ["storage", "keyword"],
68 ["keyword", "keyword"],
69 ["variable.parameter", "parameter"],
70 ["entity.name.function", "method"],
71 ["support.class.component", "method"],
72 ["support.type.primitive", "builtin"],
73 ["entity.name.type", "class"],
74 ["support.type", "class"],
75 ["support.class", "class"],
76 ["constant.language", "builtin"],
77 ["constant", "constant"],
78 ["support.constant", "constant"],
79 ["meta.parameters", "parameter"],
80 ["support.function", "method"],
81 ["entity.name.tag", "builtin"],
82 ["variable", "variable"],
83 ["meta.tag.attributes", "property"],
84 ["punctuation.definition.tag.begin.tsx", "keyword"],
85 ["punctuation.definition.tag.end.tsx", "keyword"],
86 ["punctuation", null],
87 ["meta.function-call", "method"],
88] as const;
89
90interface HighlightLinesOptions {
91 lines: string[];
92 grammar: IGrammar;
93 state: StateStack;
94 language: Language;
95}
96
97export function getStyle(scopesToCheck: string[], langugage: Language) {
98 for (const scope of scopes) {
99 if (scope[2] && scope[2] !== langugage) continue;
100 const find = scopesToCheck.find((s) => s.startsWith(scope[0]));
101 if (find) {
102 return scope[1];
103 }
104 }
105 return null;
106}
107
108function highlightLines({
109 lines,
110 grammar,
111 state,
112 language,
113}: HighlightLinesOptions) {
114 let html = "";
115 let lastHtmlStyle: string | null = null;
116 for (const line of lines) {
117 const { tokens, ruleStack, stoppedEarly } = grammar.tokenizeLine(
118 line,
119 state,
120 );
121 if (stoppedEarly) throw new Error("TODO: Tokenization stopped early?");
122 state = ruleStack;
123
124 for (const token of tokens) {
125 const str = line.slice(token.startIndex, token.endIndex);
126 if (str.trim().length === 0) {
127 // Emit but do not consider scope changes
128 html += string.escapeHtmlContent(str);
129 continue;
130 }
131
132 const style = getStyle(token.scopes, language);
133 if (style !== lastHtmlStyle) {
134 if (lastHtmlStyle) html += "</span>";
135 if (style) html += `<span class='${style}'>`;
136 }
137 html += string.escapeHtmlContent(str);
138 lastHtmlStyle = style;
139 }
140 html += "\n";
141 }
142
143 if (lastHtmlStyle) html += "</span>";
144
145 return { state, html };
146}
147
148export const getRegistry = async.once(async () => {
149 const wasmBin = await fs.readFile(
150 new URL(import.meta.resolve("vscode-oniguruma/release/onig.wasm")),
151 );
152 await oniguruma.loadWASM(wasmBin);
153
154 return new textmate.Registry({
155 onigLib: Promise.resolve({
156 createOnigScanner: (patterns) => new oniguruma.OnigScanner(patterns),
157 createOnigString: (s) => new oniguruma.OnigString(s),
158 }),
159 loadGrammar: async (scopeName: string) => {
160 for (const lang of languages) {
161 if (
162 altScopes[lang]
163 ? scopeName === altScopes[lang]
164 : scopeName.endsWith(`.${lang}`)
165 ) {
166 const file = await fs.readFile(
167 path.join(import.meta.dirname, `highlight-grammar/${lang}.plist`),
168 "utf-8",
169 );
170 return textmate.parseRawGrammar(file);
171 }
172 }
173 return null;
174 },
175 });
176});
177
178/** the registry checks its cache before an await and fills it after, so loads that overlap would each compile a grammar */
179const grammars = new Map<Language, Promise<IGrammar | null>>();
180
181export async function highlightCode(code: string, language: Language) {
182 const registry = await getRegistry();
183 const loading = grammars.get(language)
184 ?? registry.loadGrammar(altScopes[language] ?? "source." + language);
185 grammars.set(language, loading);
186 const grammar = await loading;
187 if (!grammar) {
188 throw new Error(`No grammar found for language: ${language}`);
189 }
190 let state = textmate.INITIAL;
191 const { html } = highlightLines({
192 lines: code.split("\n"),
193 grammar,
194 state,
195 language,
196 });
197 return html;
198}
199
200// export async function main() {
201// validate exts
202// for (const ext of languages) {
203// if (
204// !fs.existsSync(
205// path.join(import.meta.dirname, `highlight-grammar/${ext}.plist`),
206// )
207// ) {
208// console.error(`Missing grammar for ${ext}`);
209// }
210// // Sanity check
211// await highlightCode("wwwwwwwwwwwaaaaaaaaaaaaaaaa", ext);
212// }
213// }
214//
215// await highlightCode(
216// `return <Snow attr={'meow'}>snow!</Snow>`,
217// "tsx",
218// );
219//
220import * as fs from "#sitegen/fs";
221import * as async from "@clo/lib/async";
222import * as string from "@clo/lib/string";
223import * as path from "node:path";
224// both are CommonJS without named exports that an ES module can see
225import oniguruma from "vscode-oniguruma";
226import textmate from "vscode-textmate";
227import type { IGrammar, StateStack } from "vscode-textmate";