| ... | @@ -1,4 +1,4 @@ | ... | @@ -1,4 +1,4 @@ |
| 1 | import { createHighlighterCore, type HighlighterCore } from "@shikijs/core"; | 1 | import { createHighlighterCore, type GrammarState, type HighlighterCore } from "@shikijs/core"; |
| 2 | import { createJavaScriptRegexEngine } from "@shikijs/engine-javascript"; | 2 | import { createJavaScriptRegexEngine } from "@shikijs/engine-javascript"; |
| 3 | import bashLanguage from "@shikijs/langs/bash"; | 3 | import bashLanguage from "@shikijs/langs/bash"; |
| 4 | import javascriptLanguage from "@shikijs/langs/javascript"; | 4 | import javascriptLanguage from "@shikijs/langs/javascript"; |
| ... | @@ -50,7 +50,12 @@ const REVIEW_HIGHLIGHTER_ENGINE_PREFERENCE = resolveReviewHighlighterEnginePrefe | ... | @@ -50,7 +50,12 @@ const REVIEW_HIGHLIGHTER_ENGINE_PREFERENCE = resolveReviewHighlighterEnginePrefe |
| 50 | REVIEW_HIGHLIGHTER_ENGINE_ENV_VALUE, | 50 | REVIEW_HIGHLIGHTER_ENGINE_ENV_VALUE, |
| 51 | ); | 51 | ); |
| 52 | const REVIEW_HIGHLIGHT_CHUNK_LINE_THRESHOLD = 8; | 52 | const REVIEW_HIGHLIGHT_CHUNK_LINE_THRESHOLD = 8; |
| 53 | const REVIEW_HIGHLIGHT_CHUNK_SIZE = 200; | 53 | // Bounds each tokenizing call by characters, not lines, so long-line files |
| | 54 | // still yield to touches and renders between short batches. |
| | 55 | const REVIEW_HIGHLIGHT_CHUNK_CHARACTERS = 2_000; |
| | 56 | // A yield waits for the next frame, so batches run back to back until about one |
| | 57 | // frame of work has passed instead of yielding after every batch. |
| | 58 | const REVIEW_HIGHLIGHT_YIELD_AFTER_MS = 16; |
| 54 | const REVIEW_TOKENIZE_MAX_LINE_LENGTH = 1_000; | 59 | const REVIEW_TOKENIZE_MAX_LINE_LENGTH = 1_000; |
| 55 | const REVIEW_INITIAL_LANGUAGE_MODULES = [ | 60 | const REVIEW_INITIAL_LANGUAGE_MODULES = [ |
| 56 | bashLanguage, | 61 | bashLanguage, |
| ... | @@ -513,46 +518,53 @@ async function highlightLines( | ... | @@ -513,46 +518,53 @@ async function highlightLines( |
| 513 | const highlighter = await getHighlighter(); | 518 | const highlighter = await getHighlighter(); |
| 514 | const sourceLines = code.split("\n"); | 519 | const sourceLines = code.split("\n"); |
| 515 | const highlightedLines: Array<ReadonlyArray<ReviewHighlightedToken>> = []; | 520 | const highlightedLines: Array<ReadonlyArray<ReviewHighlightedToken>> = []; |
| 516 | const shortLineBatch: string[] = []; | 521 | // Batches resume from the previous batch's grammar state, so a comment or |
| 517 | | 522 | // template string that spans a batch boundary keeps its colors. |
| 518 | const flushShortLineBatch = async (): Promise<void> => { | 523 | let grammarState: GrammarState | undefined; |
| 519 | if (shortLineBatch.length === 0) { | 524 | let start = 0; |
| 520 | return; | 525 | let sliceStartedAt = performance.now(); |
| 521 | } | 526 | |
| 522 | | 527 | while (start < sourceLines.length) { |
| 523 | const tokenLines = highlighter.codeToTokensBase(shortLineBatch.join("\n"), { | 528 | // A skipped line leaves its ending state unknown; resume from a fresh state. |
| 524 | lang: language, | 529 | if (sourceLines[start]!.length > REVIEW_TOKENIZE_MAX_LINE_LENGTH) { |
| 525 | theme, | 530 | highlightedLines.push([{ content: sourceLines[start]!, color: null, fontStyle: null }]); |
| 526 | }); | 531 | grammarState = undefined; |
| 527 | highlightedLines.push(...normalizeHighlightedLines(tokenLines)); | 532 | start += 1; |
| 528 | shortLineBatch.length = 0; | | |
| 529 | }; | | |
| 530 | | | |
| 531 | for (let lineIndex = 0; lineIndex < sourceLines.length; lineIndex += 1) { | | |
| 532 | const line = sourceLines[lineIndex] ?? ""; | | |
| 533 | | | |
| 534 | if (line.length > REVIEW_TOKENIZE_MAX_LINE_LENGTH) { | | |
| 535 | await flushShortLineBatch(); | | |
| 536 | highlightedLines.push([{ content: line, color: null, fontStyle: null }]); | | |
| 537 | } else { | 533 | } else { |
| 538 | shortLineBatch.push(line); | 534 | let end = start; |
| 539 | } | 535 | let characters = 0; |
| | 536 | while (end < sourceLines.length) { |
| | 537 | const length = sourceLines[end]!.length; |
| | 538 | if ( |
| | 539 | length > REVIEW_TOKENIZE_MAX_LINE_LENGTH || |
| | 540 | (end > start && characters + length + 1 > REVIEW_HIGHLIGHT_CHUNK_CHARACTERS) |
| | 541 | ) { |
| | 542 | break; |
| | 543 | } |
| | 544 | characters += length + (end > start ? 1 : 0); |
| | 545 | end += 1; |
| | 546 | } |
| 540 | | 547 | |
| 541 | if (shortLineBatch.length >= REVIEW_HIGHLIGHT_CHUNK_SIZE) { | 548 | const tokenLines = highlighter.codeToTokensBase(sourceLines.slice(start, end).join("\n"), { |
| 542 | await flushShortLineBatch(); | 549 | lang: language, |
| | 550 | theme, |
| | 551 | grammarState, |
| | 552 | }); |
| | 553 | grammarState = highlighter.getLastGrammarState(tokenLines); |
| | 554 | highlightedLines.push(...normalizeHighlightedLines(tokenLines)); |
| | 555 | start = end; |
| 543 | } | 556 | } |
| 544 | | 557 | |
| 545 | if ( | 558 | if ( |
| 546 | sourceLines.length > REVIEW_HIGHLIGHT_CHUNK_LINE_THRESHOLD && | 559 | sourceLines.length > REVIEW_HIGHLIGHT_CHUNK_LINE_THRESHOLD && |
| 547 | lineIndex + 1 < sourceLines.length && | 560 | start < sourceLines.length && |
| 548 | (shortLineBatch.length === 0 || line.length > REVIEW_TOKENIZE_MAX_LINE_LENGTH) | 561 | performance.now() - sliceStartedAt >= REVIEW_HIGHLIGHT_YIELD_AFTER_MS |
| 549 | ) { | 562 | ) { |
| 550 | await waitForNextFrame(); | 563 | await waitForNextFrame(); |
| | 564 | sliceStartedAt = performance.now(); |
| 551 | } | 565 | } |
| 552 | } | 566 | } |
| 553 | | 567 | |
| 554 | await flushShortLineBatch(); | | |
| 555 | | | |
| 556 | return highlightedLines; | 568 | return highlightedLines; |
| 557 | } | 569 | } |
| 558 | | 570 | |
Julius Marminge <julius0216@outlook.com> 2026-10-07 22:48:58-07:00
GitHub <noreply@github.com> 2026-10-07 22:48:58-07:00