authorgravatar for 33724654+vedprakash2302@users.noreply.github.comVed Pandey <33724654+vedprakash2302@users.noreply.github.com> 2026-09-17 01:58:51+01:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-16 17:58:51-07:00
logc1738f131d6f0d5c6dfabb3aad46755a25319663
tree8ef79279cd1d54139d300b5d5169b02bb5dc0062
parented88937cf03e65fe5d620f170fd14b1fae6bb2e7
signature Signed by PGP key B5690EEEBB952194

fix(server): keep large sparse checkouts on the fast checkpoint path (#12154)

5 files changed, 335 insertions(+), 10 deletions(-)

apps/server/src/processRunner.ts+6-1
......@@ -25,6 +25,8 @@ export interface ProcessRunInput {
2525 readonly timeout?: Duration.Input | undefined;
2626 readonly env?: NodeJS.ProcessEnv | undefined;
2727 readonly stdin?: string | undefined;
28 /** Receives every stdout chunk, including bytes beyond the buffered output limit. */
29 readonly onStdoutChunk?: ((chunk: Uint8Array) => void) | undefined;
2830 readonly maxOutputBytes?: number | undefined;
2931 readonly outputMode?: "error" | "truncate" | undefined;
3032 readonly truncatedMarker?: string | undefined;
......@@ -327,6 +329,7 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
327329 );
328330
329331 const stdin = input.stdin;
332 const onStdoutChunk = input.onStdoutChunk;
330333 const writeStdin =
331334 stdin === undefined
332335 ? Effect.void
......@@ -352,7 +355,9 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
352355 cwd: input.cwd,
353356 spawnCwd: input.spawnCwd,
354357 streamName: "stdout",
355 stream: child.stdout,
358 stream: onStdoutChunk
359 ? child.stdout.pipe(Stream.tap((chunk) => Effect.sync(() => onStdoutChunk(chunk))))
360 : child.stdout,
356361 maxOutputBytes,
357362 outputMode,
358363 truncatedMarker,
apps/server/src/vcs/GitVcsDriver.test.ts+210
......@@ -119,6 +119,216 @@ it.effect("checkpoint capture does not rerun clean filters for unchanged indexed
119119 }).pipe(Effect.scoped, Effect.provide(GitContractLayer)),
120120);
121121
122for (const nested of [false, true]) {
123 for (const indexState of [
124 "sparse",
125 "flags",
126 "manual-skip",
127 "missing",
128 "non-cone-missing",
129 ] as const) {
130 it.effect(
131 `sparse checkpoint preserves two captures (nested=${nested}, index=${indexState})`,
132 () =>
133 Effect.gen(function* () {
134 const fs = yield* FileSystem.FileSystem;
135 const path = yield* Path.Path;
136 const driver = yield* GitVcsDriver.makeVcsDriverShape();
137 const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-sparse-" });
138 const { git } = yield* makeCheckpointFixture(driver, cwd);
139 const write = Effect.fn(function* (name: string, contents: string) {
140 yield* fs.makeDirectory(path.dirname(path.join(cwd, name)), { recursive: true });
141 yield* fs.writeFileString(path.join(cwd, name), contents);
142 });
143 for (const name of [
144 "scope/in/edit",
145 "scope/in/delete",
146 "scope/out/deep/absent",
147 "scope/out/present",
148 "elsewhere/file",
149 ]) {
150 yield* write(name, "original\n");
151 }
152 yield* git(["add", "."]);
153 yield* git(["commit", "-m", "sparse fixture"]);
154 yield* git([
155 "sparse-checkout",
156 "set",
157 "--cone",
158 "--sparse-index",
159 "scope/in",
160 "elsewhere",
161 ]);
162 if (indexState === "non-cone-missing")
163 yield* git(["sparse-checkout", "set", "--no-cone", "/scope/in/", "/elsewhere/"]);
164 yield* write("scope/in/edit", "staged\n");
165 yield* write("elsewhere/file", "staged outside\n");
166 yield* git(["add", "."]);
167 if (indexState === "flags")
168 yield* git(["update-index", "--assume-unchanged", "scope/in/delete"]);
169 if (indexState === "manual-skip")
170 yield* git(["update-index", "--skip-worktree", "scope/in/delete"]);
171 yield* git(["config", "sparse.expectFilesOutsideOfPatterns", "true"]);
172 yield* write("scope/in/edit", "working\n");
173 yield* write("scope/out/present", "modified skipped\n");
174 yield* write("scope/out/new file", "new outside cone\n");
175 yield* write("elsewhere/file", "working outside\n");
176 yield* fs.remove(path.join(cwd, "scope/in/delete"));
177 const indexPath = path.join(cwd, ".git/index");
178 if (indexState.endsWith("missing")) yield* fs.remove(indexPath);
179 const originalIndex = yield* fs
180 .readFile(indexPath)
181 .pipe(Effect.orElseSucceed(() => null));
182 const captureCwd = nested ? path.join(cwd, "scope") : cwd;
183 for (const turn of [1, 2]) {
184 const ref = CheckpointRef.make(`refs/t3/checkpoints/sparse/${turn}`);
185 if (turn === 2) {
186 yield* write("scope/in/edit", "second\n");
187 yield* fs.remove(path.join(cwd, "scope/out/new file"));
188 yield* write("scope/out/second", "second addition\n");
189 }
190 const capture = driver.checkpoints.captureCheckpoint({
191 cwd: captureCwd,
192 checkpointRef: ref,
193 });
194 if (indexState === "non-cone-missing") {
195 assert.strictEqual((yield* capture.pipe(Effect.flip))._tag, "VcsProcessExitError");
196 assert.isFalse(
197 yield* driver.checkpoints.hasCheckpointRef({ cwd: captureCwd, checkpointRef: ref }),
198 );
199 assert.isFalse(yield* fs.exists(indexPath));
200 break;
201 }
202 yield* capture;
203 for (const [name, content] of [
204 ["scope/out/deep/absent", "original\n"],
205 ["scope/out/present", "modified skipped\n"],
206 ["scope/in/edit", turn === 1 ? "working\n" : "second\n"],
207 ["elsewhere/file", nested ? "original\n" : "working outside\n"],
208 [
209 turn === 1 ? "scope/out/new file" : "scope/out/second",
210 turn === 1 ? "new outside cone\n" : "second addition\n",
211 ],
212 ]) {
213 assert.strictEqual((yield* git(["show", `${ref}:${name}`])).stdout, content);
214 }
215 const files = (yield* git(["ls-tree", "-rz", "--name-only", ref])).stdout.split("\0");
216 assert.notInclude(files, "scope/in/delete");
217 if (turn === 2) assert.notInclude(files, "scope/out/new file");
218 assert.deepEqual(
219 yield* fs.readFile(indexPath).pipe(Effect.orElseSucceed(() => null)),
220 originalIndex,
221 );
222 assert.isFalse(yield* fs.exists(path.join(cwd, "scope/out/deep/absent")));
223 assert.strictEqual(
224 yield* fs.readFileString(path.join(cwd, "elsewhere/file")),
225 "working outside\n",
226 );
227 }
228 }).pipe(Effect.scoped, Effect.provide(GitContractLayer)),
229 );
230 }
231}
232
233it.effect("checkpoint capture keeps the legacy path when Git lacks add --sparse", () =>
234 Effect.gen(function* () {
235 const fs = yield* FileSystem.FileSystem;
236 const path = yield* Path.Path;
237 const liveProcess = yield* VcsProcess.VcsProcess;
238 const driver = yield* GitVcsDriver.makeVcsDriverShape();
239 const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-legacy-" });
240 const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd);
241 yield* git(["sparse-checkout", "set", "--cone", "included"]);
242 const originalIndex = yield* fs.readFile(path.join(cwd, ".git/index"));
243 const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
244 Effect.provideService(VcsProcess.VcsProcess, {
245 run: (input) => {
246 if (input.args.includes("-h"))
247 return Effect.succeed({
248 exitCode: ChildProcessSpawner.ExitCode(129),
249 stdout: "usage: git add",
250 stderr: "",
251 stdoutTruncated: false,
252 stderrTruncated: false,
253 });
254 return liveProcess.run(
255 input.args.includes("--sparse")
256 ? {
257 ...input,
258 args: input.args.map((arg) =>
259 arg === "--sparse" ? "--unsupported-sparse" : arg,
260 ),
261 }
262 : input,
263 );
264 },
265 }),
266 );
267 yield* captureDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef });
268 assert.strictEqual((yield* git(["show", `${checkpointRef}:file.txt`])).stdout, "unstaged\n");
269 assert.deepEqual(yield* fs.readFile(path.join(cwd, ".git/index")), originalIndex);
270 }).pipe(Effect.scoped, Effect.provide(GitContractLayer)),
271);
272
273for (const indexMode of ["normal", "flags", "sparse"] as const) {
274 it.effect(
275 `checkpoint index inspection handles entries beyond the output cap (index=${indexMode})`,
276 () =>
277 Effect.gen(function* () {
278 const fs = yield* FileSystem.FileSystem;
279 const path = yield* Path.Path;
280 const liveProcess = yield* VcsProcess.VcsProcess;
281 const driver = yield* GitVcsDriver.makeVcsDriverShape();
282 const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-inspection-" });
283 const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd);
284 yield* fs.writeFileString(path.join(cwd, ".gitattributes"), "stable filter=probe\n");
285 yield* fs.writeFileString(path.join(cwd, "stable"), "unchanged\n");
286 yield* fs.writeFileString(path.join(cwd, "z-skipped"), "original\n");
287 yield* fs.makeDirectory(path.join(cwd, "excluded"));
288 yield* fs.writeFileString(path.join(cwd, "excluded/file"), "absent\n");
289 yield* fs.writeFileString(
290 path.join(cwd, ".git/filter.cjs"),
291 'require("node:fs").appendFileSync(".git/reads", "read\\n"); process.stdin.pipe(process.stdout);',
292 );
293 yield* git(["config", "filter.probe.clean", "node .git/filter.cjs"]);
294 yield* fs.utimes(path.join(cwd, "stable"), 1_700_000_000, 1_700_000_000);
295 yield* git(["add", "."]);
296 yield* git(["commit", "-m", "inspection fixture"]);
297 if (indexMode === "flags") yield* git(["update-index", "--skip-worktree", "z-skipped"]);
298 if (indexMode === "sparse")
299 yield* git(["sparse-checkout", "set", "--cone", "--sparse-index", "included"]);
300 yield* fs.writeFileString(path.join(cwd, "z-skipped"), "modified\n");
301 yield* fs.writeFileString(path.join(cwd, ".git/reads"), "");
302 const originalIndex = yield* fs.readFile(path.join(cwd, ".git/index"));
303 const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
304 Effect.provideService(VcsProcess.VcsProcess, {
305 run: (input) =>
306 liveProcess.run(
307 input.args.includes("ls-files")
308 ? {
309 ...input,
310 maxOutputBytes: 8,
311 onStdoutChunk: (chunk) => {
312 for (let i = 0; i < chunk.length; i++)
313 input.onStdoutChunk?.(chunk.subarray(i, i + 1));
314 },
315 }
316 : input,
317 ),
318 }),
319 );
320 yield* captureDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef });
321 assert.strictEqual(
322 (yield* git(["show", `${checkpointRef}:z-skipped`])).stdout,
323 "modified\n",
324 );
325 if (indexMode !== "flags")
326 assert.strictEqual(yield* fs.readFileString(path.join(cwd, ".git/reads")), "");
327 assert.deepEqual(yield* fs.readFile(path.join(cwd, ".git/index")), originalIndex);
328 }).pipe(Effect.scoped, Effect.provide(GitContractLayer)),
329 );
330}
331
122332for (const timestamp of [1_700_000_000, 1_700_000_000.9999]) {
123333 it.effect(
124334 `checkpoint capture preserves same-size edits with racy index timestamps (${timestamp})`,
apps/server/src/vcs/GitVcsDriver.ts+99-9
......@@ -1,4 +1,5 @@
11import * as NodeCrypto from "node:crypto";
2import * as NodeBuffer from "node:buffer";
23
34import * as Context from "effect/Context";
45import * as DateTime from "effect/DateTime";
......@@ -752,6 +753,12 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
752753 const checkpoints: VcsDriver.VcsCheckpointOps = {
753754 captureCheckpoint: Effect.fn("GitVcsDriver.checkpoints.captureCheckpoint")(function* (input) {
754755 const operation = "GitVcsDriver.checkpoints.captureCheckpoint";
756 const indexConfig = [
757 "-c",
758 "core.fsmonitor=false",
759 "-c",
760 "sparse.expectFilesOutsideOfPatterns=false",
761 ];
755762 const gitCommonDir = yield* resolveGitCommonDir(input.cwd);
756763 const tempIndexPath = path.join(
757764 gitCommonDir,
......@@ -772,6 +779,22 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
772779
773780 yield* Effect.gen(function* () {
774781 const headExists = yield* hasHeadCommit(input.cwd);
782 const sparseConfig = yield* execute({
783 operation,
784 cwd: input.cwd,
785 args: ["config", "--bool", "core.sparseCheckout"],
786 allowNonZeroExit: true,
787 });
788 let sparseCheckout = sparseConfig.stdout.trim() === "true";
789 if (sparseCheckout) {
790 const help = yield* execute({
791 operation,
792 cwd: input.cwd,
793 args: ["add", "-h"],
794 allowNonZeroExit: true,
795 });
796 sparseCheckout = /--(?:\[no-\])?sparse\b/.test(`${help.stdout}${help.stderr}`);
797 }
775798 if (headExists) {
776799 const reusedIndex = yield* Effect.gen(function* () {
777800 const indexPath = yield* execute({
......@@ -789,26 +812,92 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
789812 yield* execute({
790813 operation,
791814 cwd: input.cwd,
792 args: ["-c", "core.fsmonitor=false", "read-tree", "--reset", "HEAD"],
815 args: [...indexConfig, "read-tree", "--reset", "HEAD"],
793816 env: commitEnv,
794817 });
795818 // read-tree can rewrite the index, so restore its racy timestamp afterward.
796819 yield* fileSystem.utimes(tempIndexPath, indexTime, indexTime);
797 const entries = yield* execute({
820 let specialFlags = false;
821 let recordStart = true;
822 let skipped = false;
823 let skippedRecord: number[] = [];
824 const skippedPaths: string[] = [];
825 yield* vcsProcess.run({
798826 operation,
827 command: "git",
799828 cwd: input.cwd,
800 args: ["ls-files", "-v"],
829 args: [...indexConfig, "ls-files", "--full-name", "--sparse", "-v", "-z"],
801830 env: commitEnv,
802 maxOutputBytes: WORKSPACE_FILES_MAX_OUTPUT_BYTES,
831 maxOutputBytes: 4_096,
832 outputMode: "truncate",
833 // Inspect every tag; retain only skipped file paths for checking sparse rules.
834 onStdoutChunk: (chunk) => {
835 for (const byte of chunk) {
836 if (recordStart) skipped = byte === 83;
837 if (skipped && sparseCheckout) {
838 if (byte !== 0) skippedRecord.push(byte);
839 else {
840 if (skippedRecord.at(-1) !== 47) {
841 const name = Buffer.from(skippedRecord).subarray(2);
842 if (!NodeBuffer.isUtf8(name)) specialFlags = true;
843 else skippedPaths.push(name.toString("utf8"));
844 }
845 skippedRecord = [];
846 }
847 }
848 if (
849 recordStart &&
850 ((byte >= 97 && byte <= 122) || (!sparseCheckout && byte === 83))
851 ) {
852 specialFlags = true;
853 }
854 recordStart = byte === 0;
855 }
856 },
803857 });
804 // A fresh index must still capture assume-unchanged/skip-worktree files.
805 return !entries.stdoutTruncated && !/^[a-zS] /m.test(entries.stdout);
858 if (skippedPaths.length > 0 && !specialFlags) {
859 const selected = yield* execute({
860 operation,
861 cwd: input.cwd,
862 args: [...indexConfig, "sparse-checkout", "check-rules", "-z"],
863 stdin: skippedPaths.join("\0") + "\0",
864 env: commitEnv,
865 maxOutputBytes: 1,
866 outputMode: "truncate",
867 });
868 // Any selected skipped file has a manual flag, not a sparse exclusion.
869 specialFlags = selected.stdout.length > 0 || selected.stdoutTruncated;
870 }
871 // Sparse Git clears skip-worktree for present files. Manual flags still need a reset.
872 return !specialFlags;
806873 }).pipe(Effect.orElseSucceed(() => false));
807874 if (!reusedIndex) {
875 if (sparseCheckout) {
876 const cone = yield* execute({
877 operation,
878 cwd: input.cwd,
879 args: ["config", "--bool", "core.sparseCheckoutCone"],
880 allowNonZeroExit: true,
881 });
882 // Rebuilding a non-cone index loses exclusions; do not publish false deletions.
883 if (cone.stdout.trim() !== "true") {
884 return yield* new VcsProcessExitError({
885 operation,
886 command: "git read-tree",
887 cwd: input.cwd,
888 exitCode: 1,
889 detail: "Cannot rebuild a checkpoint index for non-cone sparse checkout.",
890 });
891 }
892 }
893 yield* cleanupTempIndex;
808894 yield* execute({
809895 operation,
810896 cwd: input.cwd,
811 args: ["read-tree", "HEAD"],
897 // A fresh sparse index represents excluded directories without marking them deleted.
898 args: sparseCheckout
899 ? [...indexConfig, "-c", "index.sparse=true", "read-tree", "--reset", "HEAD"]
900 : ["read-tree", "HEAD"],
812901 env: commitEnv,
813902 });
814903 }
......@@ -817,14 +906,15 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
817906 yield* execute({
818907 operation,
819908 cwd: input.cwd,
820 args: ["add", "-A", "--", "."],
909 // Preserve absent skipped entries, but capture present nonignored files outside the cone.
910 args: [...indexConfig, "add", ...(sparseCheckout ? ["--sparse"] : []), "-A", "--", "."],
821911 env: commitEnv,
822912 });
823913
824914 const writeTreeResult = yield* execute({
825915 operation,
826916 cwd: input.cwd,
827 args: ["write-tree"],
917 args: [...indexConfig, "write-tree"],
828918 env: commitEnv,
829919 });
830920 const treeOid = writeTreeResult.stdout.trim();
apps/server/src/vcs/VcsProcess.test.ts+18
......@@ -378,6 +378,24 @@ describe("VcsProcess.run", () => {
378378 }).pipe(provideLive),
379379 );
380380
381 it.effect("streams all stdout bytes beyond the buffered output cap", () =>
382 Effect.gen(function* () {
383 const chunks: Uint8Array[] = [];
384 const result = yield* run({
385 operation: "test.stream-output",
386 command: "node",
387 args: ["-e", "process.stdout.write('x'.repeat(131072) + '\\0S final\\0')"],
388 cwd: process.cwd(),
389 maxOutputBytes: 8,
390 onStdoutChunk: (chunk) => chunks.push(chunk),
391 });
392
393 expect(result.stdout).toBe("xxxxxxxx");
394 expect(result.stdoutTruncated).toBe(true);
395 expect(Buffer.concat(chunks).toString()).toBe("x".repeat(131072) + "\0S final\0");
396 }).pipe(provideLive),
397 );
398
381399 it.effect("fails with VcsProcessTimeoutError on timeout", () =>
382400 Effect.gen(function* () {
383401 const errorFiber = yield* run({
apps/server/src/vcs/VcsProcess.ts+2
......@@ -25,6 +25,7 @@ export interface VcsProcessInput {
2525 readonly cwd: string;
2626 readonly spawnCwd?: string;
2727 readonly stdin?: string;
28 readonly onStdoutChunk?: (chunk: Uint8Array) => void;
2829 readonly env?: NodeJS.ProcessEnv;
2930 readonly allowNonZeroExit?: boolean;
3031 readonly timeoutMs?: number;
......@@ -123,6 +124,7 @@ export const make = Effect.gen(function* () {
123124 cwd: input.cwd,
124125 ...(input.spawnCwd !== undefined ? { spawnCwd: input.spawnCwd } : {}),
125126 ...(input.stdin !== undefined ? { stdin: input.stdin } : {}),
127 ...(input.onStdoutChunk !== undefined ? { onStdoutChunk: input.onStdoutChunk } : {}),
126128 ...(input.env !== undefined ? { env: input.env } : {}),
127129 timeout: input.timeoutMs ?? DEFAULT_TIMEOUT_MS,
128130 maxOutputBytes: input.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES,