authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-09-04 06:50:51-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-04 06:50:51-07:00
log3b6be3ef4daa848e10c095d8a088064ac836be7f
treed09a9eb66bd0e56f406388c1f866542f86dc7008
parent082cab224624eb3a6cd494df3719c59014fb0c99
signature Signed by PGP key B5690EEEBB952194

perf(mobile): bound diff syntax highlighting work (#9673)

2 files changed, 248 insertions(+), 18 deletions(-)

apps/mobile/src/features/diffs/nativeReviewDiffHighlighter.test.ts+151-2
......@@ -1,8 +1,47 @@
1import { describe, expect, it } from "vite-plus/test";
1import { afterEach, describe, expect, it, vi } from "vite-plus/test";
22
33import type { NativeReviewDiffRow } from "./nativeReviewDiffSurface";
44import type { NativeReviewDiffFile } from "./nativeReviewDiffTypes";
5import { highlightNativeReviewDiffVisibleRows } from "./nativeReviewDiffHighlighter";
5import {
6 highlightNativeReviewDiffVisibleRows,
7 streamNativeReviewDiffTokens,
8 type NativeReviewDiffTokenChunk,
9} from "./nativeReviewDiffHighlighter";
10
11const tokenization = vi.hoisted(() => ({
12 calls: [] as string[],
13 afterCall: undefined as (() => void) | undefined,
14}));
15
16vi.mock("@shikijs/core", async (importOriginal) => {
17 const original = await importOriginal<typeof import("@shikijs/core")>();
18 return {
19 ...original,
20 createHighlighterCore: async (...args: Parameters<typeof original.createHighlighterCore>) => {
21 const highlighter = await original.createHighlighterCore(...args);
22 return {
23 ...highlighter,
24 codeToTokensBase: (...input: Parameters<typeof highlighter.codeToTokensBase>) => {
25 tokenization.calls.push(input[0]);
26 const result = highlighter.codeToTokensBase(...input);
27 tokenization.afterCall?.();
28 return result;
29 },
30 };
31 },
32 };
33});
34
35// Exercise the native entry path without requiring an iOS or Android runtime.
36vi.mock("react-native-shiki-engine", async () => {
37 const { createJavaScriptRegexEngine } = await import("@shikijs/engine-javascript");
38 return { isNativeEngineAvailable: () => true, createNativeEngine: createJavaScriptRegexEngine };
39});
40
41afterEach(() => {
42 tokenization.calls = [];
43 tokenization.afterCall = undefined;
44});
645
746const TYPESCRIPT_FILE: NativeReviewDiffFile = {
847 id: "file-1",
......@@ -187,3 +226,113 @@ describe("highlightNativeReviewDiffVisibleRows", () => {
187226 );
188227 });
189228});
229
230describe.each(["native", "javascript"] as const)("%s highlighting budgets", (engine) => {
231 const highlightRows = (rows: ReadonlyArray<NativeReviewDiffRow>, signal?: AbortSignal) =>
232 highlightNativeReviewDiffVisibleRows({
233 rows,
234 files: [TYPESCRIPT_FILE],
235 scheme: "dark",
236 engine,
237 firstRowIndex: 0,
238 lastRowIndex: rows.length - 1,
239 overscanRows: 0,
240 signal,
241 });
242
243 const line = (id: number, content: string) =>
244 makeLine({
245 id: `line-${id}`,
246 content,
247 change: "add",
248 oldLineNumber: null,
249 newLineNumber: id,
250 });
251
252 it("still highlights a line at the length limit", async () => {
253 const content = `// ${"x".repeat(997)}`;
254 const result = await highlightRows([line(1, content)]);
255
256 expect(tokenization.calls).toEqual([content]);
257 expect(result.tokensByRowId["line-1"]?.some((token) => token.color !== null)).toBe(true);
258 });
259
260 it("keeps long lines and unknown following syntax plain until the next hunk", async () => {
261 const longLine = `${"x".repeat(1_001)} /*`;
262 const rows = [
263 line(1, "export const before = 1;"),
264 line(2, longLine),
265 { kind: "comment", id: "note", commentText: "Check this", fileId: TYPESCRIPT_FILE.id },
266 line(3, "inside the comment */"),
267 makeHunk("next-hunk"),
268 line(100, "export const after = 2;"),
269 ] satisfies ReadonlyArray<NativeReviewDiffRow>;
270 const result = await highlightRows(rows);
271
272 expect(result.engine).toBe(engine);
273 expect(Object.keys(result.tokensByRowId)).toEqual(["line-1", "line-2", "line-3", "line-100"]);
274 expect(result.tokensByRowId["line-2"]).toEqual([
275 { content: longLine, color: null, fontStyle: null },
276 ]);
277 expect(result.tokensByRowId["line-3"]).toEqual([
278 { content: "inside the comment */", color: null, fontStyle: null },
279 ]);
280 expect(result.tokensByRowId["line-1"]?.some((token) => token.color !== null)).toBe(true);
281 expect(result.tokensByRowId["line-100"]?.some((token) => token.color !== null)).toBe(true);
282 expect(tokenization.calls.some((code) => code.includes(longLine))).toBe(false);
283 });
284
285 it("preserves multiline grammar and row mapping across character-limited batches", async () => {
286 const opening = line(1, "const message = `open");
287 const body = Array.from({ length: 40 }, (_, index) => line(index + 2, "inside ".repeat(45)));
288 const closing = line(42, "closed`; ");
289 const trailing = line(43, "export const after = 2;");
290 const result = await highlightRows([opening, ...body, closing, trailing]);
291 const calls = [...tokenization.calls];
292 const expected = await highlightRows([
293 opening,
294 { ...closing, newLineNumber: 2 },
295 { ...trailing, newLineNumber: 3 },
296 ]);
297
298 expect(calls.length).toBeGreaterThan(1);
299 expect(calls.every((code) => code.length <= 8_000)).toBe(true);
300 expect(result.rowCount).toBe(43);
301 for (const row of [opening, ...body, closing, trailing]) {
302 expect(result.tokensByRowId[row.id]?.map((token) => token.content).join("")).toBe(
303 row.content,
304 );
305 }
306 expect(result.tokensByRowId[closing.id]).toEqual(expected.tokensByRowId[closing.id]);
307 expect(result.tokensByRowId[trailing.id]).toEqual(expected.tokensByRowId[trailing.id]);
308 });
309
310 it("stops before later batches and publishes no partial result after cancellation", async () => {
311 const controller = new AbortController();
312 tokenization.afterCall = () => controller.abort();
313 const rows = Array.from({ length: 30 }, (_, index) => line(index + 1, `// ${"x".repeat(400)}`));
314
315 const result = await highlightRows(rows, controller.signal);
316
317 expect(tokenization.calls).toHaveLength(1);
318 expect(result.rowCount).toBe(0);
319 expect(result.tokensByRowId).toEqual({});
320 });
321
322 it("applies the same long-line guard to streamed token chunks", async () => {
323 const content = "x".repeat(10_000);
324 const chunks: NativeReviewDiffTokenChunk[] = [];
325
326 await streamNativeReviewDiffTokens({
327 rows: [line(1, content)],
328 files: [TYPESCRIPT_FILE],
329 scheme: "dark",
330 engine,
331 onChunk: (chunk) => chunks.push(chunk),
332 });
333
334 expect(chunks).toHaveLength(1);
335 expect(chunks[0]?.tokensByRowId["line-1"]).toEqual([{ content, color: null, fontStyle: null }]);
336 expect(tokenization.calls).toHaveLength(0);
337 });
338});
apps/mobile/src/features/diffs/nativeReviewDiffHighlighter.ts+97-16
......@@ -1,4 +1,4 @@
1import { createHighlighterCore, type HighlighterCore } from "@shikijs/core";
1import { createHighlighterCore, type GrammarState, type HighlighterCore } from "@shikijs/core";
22import { createJavaScriptRegexEngine } from "@shikijs/engine-javascript";
33import bashLanguage from "@shikijs/langs/bash";
44import diffLanguage from "@shikijs/langs/diff";
......@@ -46,8 +46,12 @@ export interface NativeReviewDiffHighlighterHandle {
4646 readonly engine: NativeReviewDiffHighlightEngine;
4747 readonly tokenize: (
4848 code: string,
49 options: { readonly lang: NativeReviewDiffLanguage; readonly theme: string },
50 ) => ReadonlyArray<ReadonlyArray<NativeReviewDiffToken>>;
49 options: {
50 readonly lang: NativeReviewDiffLanguage;
51 readonly theme: string;
52 readonly signal?: AbortSignal;
53 },
54 ) => Promise<ReadonlyArray<ReadonlyArray<NativeReviewDiffToken>>>;
5155}
5256
5357interface NativeReviewDiffLineRow extends NativeReviewDiffRow {
......@@ -97,6 +101,8 @@ export interface HighlightNativeReviewDiffVisibleRowsInput {
97101const NATIVE_REVIEW_DIFF_HIGHLIGHT_CHUNK_SIZE = 500;
98102const NATIVE_REVIEW_DIFF_VISIBLE_OVERSCAN_ROWS = 160;
99103const NATIVE_REVIEW_DIFF_VISIBLE_MAX_ROWS = 360;
104const NATIVE_REVIEW_DIFF_TOKENIZE_MAX_LINE_LENGTH = 1_000;
105const NATIVE_REVIEW_DIFF_TOKENIZE_MAX_CHARACTERS = 8_000;
100106
101107const NATIVE_REVIEW_DIFF_THEME_NAME_BY_SCHEME = {
102108 dark: "t3-pierre-dark",
......@@ -226,6 +232,63 @@ function normalizeTokens(
226232 );
227233}
228234
235function createHighlighterHandle(
236 highlighter: HighlighterCore,
237 engine: NativeReviewDiffHighlightEngine,
238): NativeReviewDiffHighlighterHandle {
239 return {
240 engine,
241 async tokenize(code, { lang, theme, signal }) {
242 const lines = code.split("\n");
243 const highlighted: Array<ReadonlyArray<NativeReviewDiffToken>> = [];
244 let grammarState: GrammarState | undefined;
245 let start = 0;
246
247 while (start < lines.length) {
248 if (signal?.aborted) return [];
249
250 // Skipping this line leaves its ending grammar state unknown. Keep the
251 // rest of this contiguous segment plain instead of guessing its syntax.
252 if (lines[start]!.length > NATIVE_REVIEW_DIFF_TOKENIZE_MAX_LINE_LENGTH) {
253 highlighted.push(
254 ...lines
255 .slice(start)
256 .map((content) => [{ content: content || " ", color: null, fontStyle: null }]),
257 );
258 break;
259 }
260
261 let end = start;
262 let characters = 0;
263 while (end < lines.length) {
264 const length = lines[end]!.length;
265 const nextCharacters = characters + length + (end > start ? 1 : 0);
266 if (
267 length > NATIVE_REVIEW_DIFF_TOKENIZE_MAX_LINE_LENGTH ||
268 nextCharacters > NATIVE_REVIEW_DIFF_TOKENIZE_MAX_CHARACTERS
269 ) {
270 break;
271 }
272 characters = nextCharacters;
273 end += 1;
274 }
275
276 const tokens = highlighter.codeToTokensBase(lines.slice(start, end).join("\n"), {
277 lang,
278 theme,
279 grammarState,
280 });
281 grammarState = highlighter.getLastGrammarState(tokens);
282 highlighted.push(...normalizeTokens(tokens));
283 start = end;
284 if (start < lines.length) await waitForNextFrame();
285 }
286
287 return signal?.aborted ? [] : highlighted;
288 },
289 };
290}
291
229292async function createNativeReviewDiffHighlighter(): Promise<NativeReviewDiffHighlighterHandle> {
230293 const nativeEngineModule = await import("react-native-shiki-engine");
231294 if (!nativeEngineModule.isNativeEngineAvailable()) {
......@@ -238,10 +301,7 @@ async function createNativeReviewDiffHighlighter(): Promise<NativeReviewDiffHigh
238301 engine: nativeEngineModule.createNativeEngine(),
239302 });
240303
241 return {
242 engine: "native",
243 tokenize: (code, options) => normalizeTokens(highlighter.codeToTokensBase(code, options)),
244 };
304 return createHighlighterHandle(highlighter, "native");
245305}
246306
247307async function createJavascriptReviewDiffHighlighter(): Promise<NativeReviewDiffHighlighterHandle> {
......@@ -251,10 +311,7 @@ async function createJavascriptReviewDiffHighlighter(): Promise<NativeReviewDiff
251311 engine: createJavaScriptRegexEngine(),
252312 });
253313
254 return {
255 engine: "javascript",
256 tokenize: (code, options) => normalizeTokens(highlighter.codeToTokensBase(code, options)),
257 };
314 return createHighlighterHandle(highlighter, "javascript");
258315}
259316
260317export async function getNativeReviewDiffHighlighter(
......@@ -438,8 +495,9 @@ export async function highlightNativeReviewDiffVisibleRows(
438495 const tokensByRowId: Record<string, ReadonlyArray<NativeReviewDiffToken>> = {};
439496 let segmentRows: IndexedNativeReviewDiffLineRow[] = [];
440497 let segmentFile: NativeReviewDiffFile | undefined;
498 let charactersSinceYield = 0;
441499
442 const flushSegment = () => {
500 const flushSegment = async () => {
443501 if (!segmentFile || segmentRows.length === 0 || input.signal?.aborted) {
444502 segmentRows = [];
445503 segmentFile = undefined;
......@@ -447,7 +505,20 @@ export async function highlightNativeReviewDiffVisibleRows(
447505 }
448506
449507 const code = segmentRows.map(({ row }) => row.content).join("\n");
450 const tokenLines = highlighter.tokenize(code, { lang: segmentFile.language, theme });
508 if (
509 charactersSinceYield > 0 &&
510 charactersSinceYield + code.length > NATIVE_REVIEW_DIFF_TOKENIZE_MAX_CHARACTERS
511 ) {
512 await waitForNextFrame();
513 charactersSinceYield = 0;
514 if (input.signal?.aborted) return;
515 }
516 const tokenLines = await highlighter.tokenize(code, {
517 lang: segmentFile.language,
518 theme,
519 signal: input.signal,
520 });
521 charactersSinceYield += code.length;
451522 segmentRows.forEach(({ row }, rowIndex) => {
452523 tokensByRowId[row.id] = tokenLines[rowIndex] ?? makePlainTokenFallback(row);
453524 });
......@@ -456,6 +527,7 @@ export async function highlightNativeReviewDiffVisibleRows(
456527 };
457528
458529 for (const selectedRow of selectedRows) {
530 if (input.signal?.aborted) break;
459531 const { row } = selectedRow;
460532 const file = fileMap.get(row.fileId);
461533 if (!file) {
......@@ -469,13 +541,17 @@ export async function highlightNativeReviewDiffVisibleRows(
469541 (previousRow !== undefined &&
470542 !canShareGrammarContext(previousRow, selectedRow, input.rows)))
471543 ) {
472 flushSegment();
544 await flushSegment();
473545 }
474546
475547 segmentFile = file;
476548 segmentRows.push(selectedRow);
477549 }
478 flushSegment();
550 await flushSegment();
551
552 if (input.signal?.aborted) {
553 return { engine: highlighter.engine, tokensByRowId: {}, rowCount: 0, durationMs: 0 };
554 }
479555
480556 return {
481557 engine: highlighter.engine,
......@@ -504,7 +580,12 @@ export async function streamNativeReviewDiffTokens(
504580 const startedAt = performance.now();
505581 const chunkRows = fileRows.slice(startIndex, startIndex + chunkSize);
506582 const code = chunkRows.map((row) => row.content).join("\n");
507 const tokenLines = highlighter.tokenize(code, { lang: file.language, theme });
583 const tokenLines = await highlighter.tokenize(code, {
584 lang: file.language,
585 theme,
586 signal: input.signal,
587 });
588 if (input.signal?.aborted) return highlighter.engine;
508589 const tokensByRowId: Record<string, ReadonlyArray<NativeReviewDiffToken>> = {};
509590
510591 chunkRows.forEach((row, rowIndex) => {