authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-02-17 16:01:13-08:00
committergravatar for jmarminge@gmail.comJulius Marminge <jmarminge@gmail.com> 2026-02-17 19:47:09-08:00
log5d61b8b14f7f80e6183a7cc9b51a1c48802ead34
tree80c50918c36834e909bb358e895ce61398814156
parent2743e45dbed2895e47aed18bbabb36e8dae54247

Add tests for chunking git check-ignore in workspace entry search

- Introduce a new test file to validate chunking behavior of git check-ignore to prevent oversized input strings. - Adjust workspaceEntries.ts to implement chunking logic for git check-ignore, improving performance with large ignored path lists. - Set a maximum byte limit for stdin in git check-ignore to enhance stability and prevent errors during processing.

2 files changed, 145 insertions(+), 31 deletions(-)

apps/server/src/workspaceEntries.chunking.test.ts created+80
...@@ -0,0 +1,80 @@
1import { assert, beforeEach, describe, it, vi } from "vitest";
2import type { ProcessRunOptions, ProcessRunResult } from "./processRunner";
3
4const { runProcessMock } = vi.hoisted(() => ({
5 runProcessMock: vi.fn<
6 (
7 command: string,
8 args: readonly string[],
9 options?: ProcessRunOptions,
10 ) => Promise<ProcessRunResult>
11 >(),
12}));
13
14vi.mock("./processRunner", () => ({
15 runProcess: runProcessMock,
16}));
17
18function processResult(
19 overrides: Partial<ProcessRunResult> & Pick<ProcessRunResult, "stdout" | "code">,
20): ProcessRunResult {
21 return {
22 stdout: overrides.stdout,
23 code: overrides.code,
24 stderr: overrides.stderr ?? "",
25 signal: overrides.signal ?? null,
26 timedOut: overrides.timedOut ?? false,
27 stdoutTruncated: overrides.stdoutTruncated ?? false,
28 stderrTruncated: overrides.stderrTruncated ?? false,
29 };
30}
31
32describe("searchWorkspaceEntries git-ignore chunking", () => {
33 beforeEach(() => {
34 runProcessMock.mockReset();
35 vi.resetModules();
36 });
37
38 it("chunks git check-ignore stdin to avoid building giant strings", async () => {
39 const ignoredPaths = Array.from(
40 { length: 5000 },
41 (_, index) => `ignored/${index.toString().padStart(5, "0")}/${"x".repeat(80)}.ts`,
42 );
43 const keptPaths = ["src/keep.ts", "docs/readme.md"];
44 const listedPaths = [...ignoredPaths, ...keptPaths];
45 let checkIgnoreCalls = 0;
46
47 runProcessMock.mockImplementation(async (_command, args, options) => {
48 if (args[0] === "rev-parse") {
49 return processResult({ code: 0, stdout: "true\n" });
50 }
51
52 if (args[0] === "ls-files") {
53 return processResult({ code: 0, stdout: `${listedPaths.join("\0")}\0` });
54 }
55
56 if (args[0] === "check-ignore") {
57 checkIgnoreCalls += 1;
58 const chunkPaths = (options?.stdin ?? "").split("\0").filter((value) => value.length > 0);
59 const chunkIgnored = chunkPaths.filter((value) => value.startsWith("ignored/"));
60 return processResult({
61 code: chunkIgnored.length > 0 ? 0 : 1,
62 stdout: chunkIgnored.length > 0 ? `${chunkIgnored.join("\0")}\0` : "",
63 });
64 }
65
66 throw new Error(`Unexpected command: git ${args.join(" ")}`);
67 });
68
69 const { searchWorkspaceEntries } = await import("./workspaceEntries");
70 const result = await searchWorkspaceEntries({
71 cwd: "/virtual/workspace",
72 query: "",
73 limit: 100,
74 });
75
76 assert.isAbove(checkIgnoreCalls, 1);
77 assert.isFalse(result.entries.some((entry) => entry.path.startsWith("ignored/")));
78 assert.isTrue(result.entries.some((entry) => entry.path === "src/keep.ts"));
79 });
80});
apps/server/src/workspaceEntries.ts+65-31
...@@ -14,17 +14,8 @@ const WORKSPACE_CACHE_TTL_MS = 15_000;...@@ -14,17 +14,8 @@ const WORKSPACE_CACHE_TTL_MS = 15_000;
14const WORKSPACE_CACHE_MAX_KEYS = 4;14const WORKSPACE_CACHE_MAX_KEYS = 4;
15const WORKSPACE_INDEX_MAX_ENTRIES = 25_000;15const WORKSPACE_INDEX_MAX_ENTRIES = 25_000;
16const WORKSPACE_SCAN_READDIR_CONCURRENCY = 32;16const WORKSPACE_SCAN_READDIR_CONCURRENCY = 32;
17const IGNORED_DIRECTORY_NAMES = new Set([17const GIT_CHECK_IGNORE_MAX_STDIN_BYTES = 256 * 1024;
18 ".git",18const IGNORED_DIRECTORY_NAMES = new Set([".git", "node_modules"]);
19 ".convex",
20 "node_modules",
21 ".next",
22 ".turbo",
23 "dist",
24 "build",
25 "out",
26 ".cache",
27]);
2819
29interface WorkspaceIndex {20interface WorkspaceIndex {
30 scannedAt: number;21 scannedAt: number;
...@@ -56,7 +47,10 @@ function basenameOf(input: string): string {...@@ -56,7 +47,10 @@ function basenameOf(input: string): string {
56}47}
5748
58function normalizeQuery(input: string): string {49function normalizeQuery(input: string): string {
59 return input.trim().replace(/^[@./]+/, "").toLowerCase();50 return input
51 .trim()
52 .replace(/^[@./]+/, "")
53 .toLowerCase();
60}54}
6155
62function scoreEntry(entry: ProjectEntry, query: string): number {56function scoreEntry(entry: ProjectEntry, query: string): number {
...@@ -145,26 +139,67 @@ async function filterGitIgnoredPaths(cwd: string, relativePaths: string[]): Prom...@@ -145,26 +139,67 @@ async function filterGitIgnoredPaths(cwd: string, relativePaths: string[]): Prom
145 return relativePaths;139 return relativePaths;
146 }140 }
147141
148 const checkIgnore = await runProcess("git", ["check-ignore", "--no-index", "-z", "--stdin"], {142 const ignoredPaths = new Set<string>();
149 cwd,143 let chunk: string[] = [];
150 allowNonZeroExit: true,144 let chunkBytes = 0;
151 timeoutMs: 20_000,145
152 maxBufferBytes: 16 * 1024 * 1024,146 const flushChunk = async (): Promise<boolean> => {
153 outputMode: "truncate",147 if (chunk.length === 0) {
154 stdin: `${relativePaths.join("\0")}\0`,148 return true;
155 }).catch(() => null);149 }
156 if (!checkIgnore) {150
157 return relativePaths;151 const checkIgnore = await runProcess("git", ["check-ignore", "--no-index", "-z", "--stdin"], {
152 cwd,
153 allowNonZeroExit: true,
154 timeoutMs: 20_000,
155 maxBufferBytes: 16 * 1024 * 1024,
156 outputMode: "truncate",
157 stdin: `${chunk.join("\0")}\0`,
158 }).catch(() => null);
159 chunk = [];
160 chunkBytes = 0;
161
162 if (!checkIgnore) {
163 return false;
164 }
165
166 // git-check-ignore exits with 1 when no paths match.
167 if (checkIgnore.code !== 0 && checkIgnore.code !== 1) {
168 return false;
169 }
170
171 const matchedIgnoredPaths = splitNullSeparatedPaths(
172 checkIgnore.stdout,
173 Boolean(checkIgnore.stdoutTruncated),
174 );
175 for (const ignoredPath of matchedIgnoredPaths) {
176 ignoredPaths.add(ignoredPath);
177 }
178 return true;
179 };
180
181 for (const relativePath of relativePaths) {
182 const relativePathBytes = Buffer.byteLength(relativePath) + 1;
183 if (
184 chunk.length > 0 &&
185 chunkBytes + relativePathBytes > GIT_CHECK_IGNORE_MAX_STDIN_BYTES &&
186 !(await flushChunk())
187 ) {
188 return relativePaths;
189 }
190
191 chunk.push(relativePath);
192 chunkBytes += relativePathBytes;
193
194 if (chunkBytes >= GIT_CHECK_IGNORE_MAX_STDIN_BYTES && !(await flushChunk())) {
195 return relativePaths;
196 }
158 }197 }
159198
160 // git-check-ignore exits with 1 when no paths match.199 if (!(await flushChunk())) {
161 if (checkIgnore.code !== 0 && checkIgnore.code !== 1) {
162 return relativePaths;200 return relativePaths;
163 }201 }
164202
165 const ignoredPaths = new Set(
166 splitNullSeparatedPaths(checkIgnore.stdout, Boolean(checkIgnore.stdoutTruncated)),
167 );
168 if (ignoredPaths.size === 0) {203 if (ignoredPaths.size === 0) {
169 return relativePaths;204 return relativePaths;
170 }205 }
...@@ -210,14 +245,14 @@ async function buildWorkspaceIndexFromGit(cwd: string): Promise<WorkspaceIndex |...@@ -210,14 +245,14 @@ async function buildWorkspaceIndexFromGit(cwd: string): Promise<WorkspaceIndex |
210 }245 }
211246
212 const directoryEntries: ProjectEntry[] = [...directorySet]247 const directoryEntries: ProjectEntry[] = [...directorySet]
213 .sort((left, right) => left.localeCompare(right))248 .toSorted((left, right) => left.localeCompare(right))
214 .map((directoryPath) => ({249 .map((directoryPath) => ({
215 path: directoryPath,250 path: directoryPath,
216 kind: "directory",251 kind: "directory",
217 parentPath: parentPathOf(directoryPath),252 parentPath: parentPathOf(directoryPath),
218 }));253 }));
219 const fileEntries: ProjectEntry[] = [...new Set(filePaths)]254 const fileEntries: ProjectEntry[] = [...new Set(filePaths)]
220 .sort((left, right) => left.localeCompare(right))255 .toSorted((left, right) => left.localeCompare(right))
221 .map((filePath) => ({256 .map((filePath) => ({
222 path: filePath,257 path: filePath,
223 kind: "file",258 kind: "file",
...@@ -228,8 +263,7 @@ async function buildWorkspaceIndexFromGit(cwd: string): Promise<WorkspaceIndex |...@@ -228,8 +263,7 @@ async function buildWorkspaceIndexFromGit(cwd: string): Promise<WorkspaceIndex |
228 return {263 return {
229 scannedAt: Date.now(),264 scannedAt: Date.now(),
230 entries: entries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES),265 entries: entries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES),
231 truncated:266 truncated: Boolean(listedFiles.stdoutTruncated) || entries.length > WORKSPACE_INDEX_MAX_ENTRIES,
232 Boolean(listedFiles.stdoutTruncated) || entries.length > WORKSPACE_INDEX_MAX_ENTRIES,
233 };267 };
234}268}
235269