authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-07 22:48:58-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-07 22:48:58-07:00
log33afd5a5aa361890c9ce22479e8f52f6b18248aa
tree1fa2782d333535f5fc752fb374e9d39576419a4d
parentb402f5fc65f6f4554266dea385a7da7ff353f028
signature Signed by PGP key B5690EEEBB952194

perf(mobile): highlight source files in small batches that keep grammar state (#16729)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

2 files changed, 61 insertions(+), 31 deletions(-)

apps/mobile/src/features/review/shikiReviewHighlighter.test.ts+18
......@@ -27,6 +27,24 @@ describe("highlightSourceFile", () => {
2727 );
2828 });
2929
30 it("keeps colors for a block comment that spans highlight batches", async () => {
31 // Long enough that the comment body crosses at least one batch boundary.
32 const body = Array.from({ length: 400 }, (_, index) => `const insideComment${index} = 1;`);
33 const highlighted = await highlightSourceFile({
34 path: "example.ts",
35 contents: ["/*", ...body, "*/", "const after = 2;"].join("\n"),
36 theme: "dark",
37 });
38
39 const commentColors = new Set(
40 highlighted.slice(1, -2).flatMap((line) => line.map((token) => token.color)),
41 );
42 expect(commentColors.size).toBe(1);
43 expect(highlighted.at(-1)?.map((token) => token.color)).not.toEqual(
44 highlighted[1]?.map((token) => token.color),
45 );
46 });
47
3048 it("falls back to plain tokens for very long lines", async () => {
3149 const longLine = `const value = "${"a".repeat(1_100)}";`;
3250
apps/mobile/src/features/review/shikiReviewHighlighter.ts+43-31
......@@ -1,4 +1,4 @@
1import { createHighlighterCore, type HighlighterCore } from "@shikijs/core";
1import { createHighlighterCore, type GrammarState, type HighlighterCore } from "@shikijs/core";
22import { createJavaScriptRegexEngine } from "@shikijs/engine-javascript";
33import bashLanguage from "@shikijs/langs/bash";
44import javascriptLanguage from "@shikijs/langs/javascript";
......@@ -50,7 +50,12 @@ const REVIEW_HIGHLIGHTER_ENGINE_PREFERENCE = resolveReviewHighlighterEnginePrefe
5050 REVIEW_HIGHLIGHTER_ENGINE_ENV_VALUE,
5151);
5252const REVIEW_HIGHLIGHT_CHUNK_LINE_THRESHOLD = 8;
53const REVIEW_HIGHLIGHT_CHUNK_SIZE = 200;
53// Bounds each tokenizing call by characters, not lines, so long-line files
54// still yield to touches and renders between short batches.
55const REVIEW_HIGHLIGHT_CHUNK_CHARACTERS = 2_000;
56// A yield waits for the next frame, so batches run back to back until about one
57// frame of work has passed instead of yielding after every batch.
58const REVIEW_HIGHLIGHT_YIELD_AFTER_MS = 16;
5459const REVIEW_TOKENIZE_MAX_LINE_LENGTH = 1_000;
5560const REVIEW_INITIAL_LANGUAGE_MODULES = [
5661 bashLanguage,
......@@ -513,46 +518,53 @@ async function highlightLines(
513518 const highlighter = await getHighlighter();
514519 const sourceLines = code.split("\n");
515520 const highlightedLines: Array<ReadonlyArray<ReviewHighlightedToken>> = [];
516 const shortLineBatch: string[] = [];
517
518 const flushShortLineBatch = async (): Promise<void> => {
519 if (shortLineBatch.length === 0) {
520 return;
521 }
522
523 const tokenLines = highlighter.codeToTokensBase(shortLineBatch.join("\n"), {
524 lang: language,
525 theme,
526 });
527 highlightedLines.push(...normalizeHighlightedLines(tokenLines));
528 shortLineBatch.length = 0;
529 };
530
531 for (let lineIndex = 0; lineIndex < sourceLines.length; lineIndex += 1) {
532 const line = sourceLines[lineIndex] ?? "";
533
534 if (line.length > REVIEW_TOKENIZE_MAX_LINE_LENGTH) {
535 await flushShortLineBatch();
536 highlightedLines.push([{ content: line, color: null, fontStyle: null }]);
521 // Batches resume from the previous batch's grammar state, so a comment or
522 // template string that spans a batch boundary keeps its colors.
523 let grammarState: GrammarState | undefined;
524 let start = 0;
525 let sliceStartedAt = performance.now();
526
527 while (start < sourceLines.length) {
528 // A skipped line leaves its ending state unknown; resume from a fresh state.
529 if (sourceLines[start]!.length > REVIEW_TOKENIZE_MAX_LINE_LENGTH) {
530 highlightedLines.push([{ content: sourceLines[start]!, color: null, fontStyle: null }]);
531 grammarState = undefined;
532 start += 1;
537533 } else {
538 shortLineBatch.push(line);
539 }
534 let end = start;
535 let characters = 0;
536 while (end < sourceLines.length) {
537 const length = sourceLines[end]!.length;
538 if (
539 length > REVIEW_TOKENIZE_MAX_LINE_LENGTH ||
540 (end > start && characters + length + 1 > REVIEW_HIGHLIGHT_CHUNK_CHARACTERS)
541 ) {
542 break;
543 }
544 characters += length + (end > start ? 1 : 0);
545 end += 1;
546 }
540547
541 if (shortLineBatch.length >= REVIEW_HIGHLIGHT_CHUNK_SIZE) {
542 await flushShortLineBatch();
548 const tokenLines = highlighter.codeToTokensBase(sourceLines.slice(start, end).join("\n"), {
549 lang: language,
550 theme,
551 grammarState,
552 });
553 grammarState = highlighter.getLastGrammarState(tokenLines);
554 highlightedLines.push(...normalizeHighlightedLines(tokenLines));
555 start = end;
543556 }
544557
545558 if (
546559 sourceLines.length > REVIEW_HIGHLIGHT_CHUNK_LINE_THRESHOLD &&
547 lineIndex + 1 < sourceLines.length &&
548 (shortLineBatch.length === 0 || line.length > REVIEW_TOKENIZE_MAX_LINE_LENGTH)
560 start < sourceLines.length &&
561 performance.now() - sliceStartedAt >= REVIEW_HIGHLIGHT_YIELD_AFTER_MS
549562 ) {
550563 await waitForNextFrame();
564 sliceStartedAt = performance.now();
551565 }
552566 }
553567
554 await flushShortLineBatch();
555
556568 return highlightedLines;
557569}
558570