| 1 | import type { HighlighterCore } from "@shikijs/core"; |
| 2 | |
| 3 | /** A code block owns this session. Only completed lines survive the next update. */ |
| 4 | export function createIncrementalSnippet( |
| 5 | highlighter: HighlighterCore, |
| 6 | language: string, |
| 7 | theme: string, |
| 8 | ) { |
| 9 | const options = { lang: language, theme }; |
| 10 | let cached: |
| 11 | | { |
| 12 | prefix: string; |
| 13 | tokens: ReturnType<HighlighterCore["codeToTokensBase"]>; |
| 14 | state: ReturnType<HighlighterCore["getLastGrammarState"]>; |
| 15 | } |
| 16 | | undefined; |
| 17 | let revision = 0; |
| 18 | |
| 19 | function* tokenize(code: string) { |
| 20 | const currentRevision = ++revision; |
| 21 | const previous = cached && code.startsWith(cached.prefix) ? cached : undefined; |
| 22 | const end = code.lastIndexOf("\n") + 1; |
| 23 | let state = previous?.state; |
| 24 | const tokens = [...(previous?.tokens ?? [])]; |
| 25 | const completed = code.slice(previous?.prefix.length ?? 0, end); |
| 26 | if (completed) { |
| 27 | const lines = completed.slice(0, -1).split("\n"); |
| 28 | for (let offset = 0; offset < lines.length; offset += 200) { |
| 29 | const batch = highlighter.codeToTokensBase(lines.slice(offset, offset + 200).join("\n"), { |
| 30 | ...options, |
| 31 | ...(state ? { grammarState: state } : {}), |
| 32 | }); |
| 33 | state = highlighter.getLastGrammarState(batch); |
| 34 | tokens.push(...batch); |
| 35 | if (offset + 200 < lines.length) yield; |
| 36 | } |
| 37 | } |
| 38 | if (currentRevision === revision) { |
| 39 | cached = state ? { prefix: code.slice(0, end), tokens, state } : undefined; |
| 40 | } |
| 41 | return [ |
| 42 | ...tokens, |
| 43 | ...highlighter.codeToTokensBase(code.slice(end), { |
| 44 | ...options, |
| 45 | ...(state ? { grammarState: state } : {}), |
| 46 | }), |
| 47 | ]; |
| 48 | } |
| 49 | const highlight = async (code: string) => { |
| 50 | const work = tokenize(code); |
| 51 | let next = work.next(); |
| 52 | while (!next.done) { |
| 53 | await new Promise<void>((resolve) => setTimeout(resolve, 0)); |
| 54 | next = work.next(); |
| 55 | } |
| 56 | return next.value; |
| 57 | }; |
| 58 | // A small append can finish during render, avoiding a plain-text commit before |
| 59 | // the asynchronous effect supplies colors. Cold/large changes stay asynchronous. |
| 60 | highlight.read = (code: string) => { |
| 61 | if (!cached || !code.startsWith(cached.prefix)) return undefined; |
| 62 | const tail = code.slice(cached.prefix.length); |
| 63 | if (tail.length > 2_000 || tail.split("\n").length > 200) return undefined; |
| 64 | const next = tokenize(code).next(); |
| 65 | return next.done ? next.value : undefined; |
| 66 | }; |
| 67 | return highlight; |
| 68 | } |