1import { describe, expect, it } from "vite-plus/test";
2
3import { embeddedScripts } from "../../lib/embeddedScripts";
4import { keyedLines, withEmbeddedScripts, wordsOf } from "./HighlightedTokens";
5
6function tokensOf(...contents: string[]) {
7 let offset = 0;
8 return contents.map((content) => {
9 const token = { content, offset };
10 offset += content.length;
11 return token;
12 });
13}
14
15function shape(parts: ReturnType<typeof wordsOf>) {
16 return parts.map((part) =>
17 typeof part === "string" ? part : part.pieces.map((piece) => piece.text).join(""),
18 );
19}
20
21describe("wordsOf", () => {
22 it.each([
23 // Shiki splits these shell words into several tokens.
24 [["echo", " ", '"', "$HOME", "/${", "PATH", "%%:*", '}"'], 'echo "$HOME/${PATH%%:*}"'],
25 [["rsync", " ", "-a", " ", "--exclude", " ", "build"], "rsync -a --exclude build"],
26 // A token can carry whitespace on either side or inside a string.
27 [["git", " && ", "echo", " ", '"two words"', "; "], 'git && echo "two words"; '],
28 [[" ", "indented"], " indented"],
29 [[], ""],
30 ])("wraps the same words as the plain-text fallback: %j", (contents, line) => {
31 expect(shape(wordsOf(tokensOf(...contents)))).toEqual(
32 line.split(/(\s+)/u).filter((part) => part !== ""),
33 );
34 });
35
36 it("keeps each piece's own token so colors survive regrouping", () => {
37 const [word] = wordsOf(tokensOf('"', "$HOME", '"'));
38 expect(
39 typeof word === "string" ? null : word?.pieces.map((piece) => piece.token.content),
40 ).toEqual(['"', "$HOME", '"']);
41 });
42});
43
44describe("keyedLines", () => {
45 it.each([
46 "git status\ngit diff",
47 "Get-ChildItem\r\nWrite-Output done\r\n",
48 "mixed\r\nendings\n\nlone \r stays",
49 ])("renders the same text as the source: %j", (code) => {
50 // Shiki splits lines on \r?\n and drops the ending from the tokens.
51 const lines = code.split(/\r?\n/u).map((line) => tokensOf(line));
52 const rendered = keyedLines(code, lines)
53 .map(({ tokens, ending }) => tokens.map((token) => token.content).join("") + ending)
54 .join("");
55 expect(rendered).toBe(code);
56 });
57
58 it("keys empty lines apart", () => {
59 const code = "a\r\n\r\n\r\nb";
60 const keys = keyedLines(
61 code,
62 code.split(/\r?\n/u).map((line) => tokensOf(line)),
63 ).map((line) => line.key);
64 expect(new Set(keys).size).toBe(keys.length);
65 });
66});
67
68describe("withEmbeddedScripts", () => {
69 // A stand-in grammar: each word is one token colored with the language name,
70 // and lines split the way Shiki splits them.
71 function tokenize(text: string, language: string) {
72 if (language === "unknown") throw new Error("Language not found");
73 let lineStart = 0;
74 return text.split(/\r?\n/u).map((line) => {
75 const tokens = [...line.matchAll(/\S+|\s+/gu)].map((match) => ({
76 content: match[0],
77 offset: lineStart + match.index,
78 color: language,
79 }));
80 lineStart +=
81 line.length + (text.slice(lineStart + line.length).match(/^\r?\n/u)?.[0].length ?? 0);
82 return tokens;
83 });
84 }
85
86 function colorsOf(code: string, scripts = embeddedScripts(code)) {
87 const lines = withEmbeddedScripts(code, tokenize(code, "shellscript"), scripts, tokenize);
88 expect(lines.map((line) => line.map((token) => token.content).join("")).join("\n")).toBe(
89 code.replaceAll("\r\n", "\n"),
90 );
91 return lines.flat().map((token) => [token.content, token.color]);
92 }
93
94 it("colors each nested script with its own grammar, keeping the text", () => {
95 expect(colorsOf(`bash -lc "python3 -c 'print(1)'"`)).toEqual([
96 [`bash -lc "python3 -c '`, "shellscript"],
97 ["print(1)", "python"],
98 [`'"`, "shellscript"],
99 ]);
100 });
101
102 it("colors an escape sequence like the character it spells", () => {
103 expect(colorsOf(`bash -c "python3 -c 'print(\\"x\\")'"`)).toContainEqual([
104 `print(\\"x\\")`,
105 "python",
106 ]);
107 });
108
109 it("keeps the surrounding colors where a grammar is missing", () => {
110 const code = "cat > x.ts <<'EOF'\r\nbody\r\nEOF";
111 const [script] = embeddedScripts(code);
112 // Compare per character: untouched lines keep their grammar's token boundaries.
113 const characterColors = (scripts: ReturnType<typeof embeddedScripts>) =>
114 colorsOf(code, scripts).flatMap(([content, color]) => [...content!].map(() => color));
115 expect(characterColors([{ ...script!, language: "unknown" }])).toEqual(characterColors([]));
116 });
117});