authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-03-28 20:50:23-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-03-28 20:50:23-07:00
log9e60597174dcd69ae405d3966a374213c745388a
tree45c2b4470177339298bfb8387006b64f7fe8a87d
parentafc807ac3f8cc827d5f30f587a599dd077f244ff
signature Signed by PGP key B5690EEEBB952194

Truncate oversized git diffs instead of failing (#1499)

4 files changed, 239 insertions(+), 19 deletions(-)

apps/server/src/checkpointing/Layers/CheckpointStore.test.ts created+122
...@@ -0,0 +1,122 @@
1import path from "node:path";
2
3import * as NodeServices from "@effect/platform-node/NodeServices";
4import { it } from "@effect/vitest";
5import { Effect, FileSystem, Layer, PlatformError, Scope } from "effect";
6import { describe, expect } from "vitest";
7
8import { checkpointRefForThreadTurn } from "../Utils.ts";
9import { CheckpointStoreLive } from "./CheckpointStore.ts";
10import { CheckpointStore } from "../Services/CheckpointStore.ts";
11import { GitCoreLive } from "../../git/Layers/GitCore.ts";
12import { GitCore } from "../../git/Services/GitCore.ts";
13import { GitCommandError } from "../../git/Errors.ts";
14import { ServerConfig } from "../../config.ts";
15import { ThreadId } from "@t3tools/contracts";
16
17const ServerConfigLayer = ServerConfig.layerTest(process.cwd(), {
18 prefix: "t3-checkpoint-store-test-",
19});
20const GitCoreTestLayer = GitCoreLive.pipe(
21 Layer.provide(ServerConfigLayer),
22 Layer.provide(NodeServices.layer),
23);
24const CheckpointStoreTestLayer = CheckpointStoreLive.pipe(
25 Layer.provide(GitCoreTestLayer),
26 Layer.provide(NodeServices.layer),
27);
28const TestLayer = Layer.mergeAll(NodeServices.layer, GitCoreTestLayer, CheckpointStoreTestLayer);
29
30function makeTmpDir(
31 prefix = "checkpoint-store-test-",
32): Effect.Effect<string, PlatformError.PlatformError, FileSystem.FileSystem | Scope.Scope> {
33 return Effect.gen(function* () {
34 const fileSystem = yield* FileSystem.FileSystem;
35 return yield* fileSystem.makeTempDirectoryScoped({ prefix });
36 });
37}
38
39function writeTextFile(
40 filePath: string,
41 contents: string,
42): Effect.Effect<void, PlatformError.PlatformError, FileSystem.FileSystem> {
43 return Effect.gen(function* () {
44 const fileSystem = yield* FileSystem.FileSystem;
45 yield* fileSystem.writeFileString(filePath, contents);
46 });
47}
48
49function git(
50 cwd: string,
51 args: ReadonlyArray<string>,
52): Effect.Effect<string, GitCommandError, GitCore> {
53 return Effect.gen(function* () {
54 const gitCore = yield* GitCore;
55 const result = yield* gitCore.execute({
56 operation: "CheckpointStore.test.git",
57 cwd,
58 args,
59 timeoutMs: 10_000,
60 });
61 return result.stdout.trim();
62 });
63}
64
65function initRepoWithCommit(
66 cwd: string,
67): Effect.Effect<
68 void,
69 GitCommandError | PlatformError.PlatformError,
70 GitCore | FileSystem.FileSystem
71> {
72 return Effect.gen(function* () {
73 const core = yield* GitCore;
74 yield* core.initRepo({ cwd });
75 yield* git(cwd, ["config", "user.email", "test@test.com"]);
76 yield* git(cwd, ["config", "user.name", "Test"]);
77 yield* writeTextFile(path.join(cwd, "README.md"), "# test\n");
78 yield* git(cwd, ["add", "."]);
79 yield* git(cwd, ["commit", "-m", "initial commit"]);
80 });
81}
82
83function buildLargeText(lineCount = 20_000): string {
84 return Array.from({ length: lineCount }, (_, index) => `line ${String(index).padStart(5, "0")}`)
85 .join("\n")
86 .concat("\n");
87}
88
89it.layer(TestLayer)("CheckpointStoreLive", (it) => {
90 describe("diffCheckpoints", () => {
91 it.effect("returns full oversized checkpoint diffs without truncation", () =>
92 Effect.gen(function* () {
93 const tmp = yield* makeTmpDir();
94 yield* initRepoWithCommit(tmp);
95 const checkpointStore = yield* CheckpointStore;
96 const threadId = ThreadId.makeUnsafe("thread-checkpoint-store");
97 const fromCheckpointRef = checkpointRefForThreadTurn(threadId, 0);
98 const toCheckpointRef = checkpointRefForThreadTurn(threadId, 1);
99
100 yield* checkpointStore.captureCheckpoint({
101 cwd: tmp,
102 checkpointRef: fromCheckpointRef,
103 });
104 yield* writeTextFile(path.join(tmp, "README.md"), buildLargeText());
105 yield* checkpointStore.captureCheckpoint({
106 cwd: tmp,
107 checkpointRef: toCheckpointRef,
108 });
109
110 const diff = yield* checkpointStore.diffCheckpoints({
111 cwd: tmp,
112 fromCheckpointRef,
113 toCheckpointRef,
114 });
115
116 expect(diff).toContain("diff --git");
117 expect(diff).not.toContain("[truncated]");
118 expect(diff).toContain("+line 19999");
119 }),
120 );
121 });
122});
apps/server/src/git/Layers/GitCore.test.ts+40
...@@ -132,6 +132,12 @@ function commitWithDate(...@@ -132,6 +132,12 @@ function commitWithDate(
132 });132 });
133}133}
134134
135function buildLargeText(lineCount = 20_000): string {
136 return Array.from({ length: lineCount }, (_, index) => `line ${String(index).padStart(5, "0")}`)
137 .join("\n")
138 .concat("\n");
139}
140
135// ── Tests ──141// ── Tests ──
136142
137it.layer(TestLayer)("git integration", (it) => {143it.layer(TestLayer)("git integration", (it) => {
...@@ -1670,6 +1676,40 @@ it.layer(TestLayer)("git integration", (it) => {...@@ -1670,6 +1676,40 @@ it.layer(TestLayer)("git integration", (it) => {
1670 }),1676 }),
1671 );1677 );
16721678
1679 it.effect("prepareCommitContext truncates oversized staged patches instead of failing", () =>
1680 Effect.gen(function* () {
1681 const tmp = yield* makeTmpDir();
1682 yield* initRepoWithCommit(tmp);
1683 const core = yield* GitCore;
1684
1685 yield* writeTextFile(path.join(tmp, "README.md"), buildLargeText());
1686
1687 const context = yield* core.prepareCommitContext(tmp);
1688 expect(context).not.toBeNull();
1689 expect(context!.stagedSummary).toContain("README.md");
1690 expect(context!.stagedPatch).toContain("[truncated]");
1691 }),
1692 );
1693
1694 it.effect("readRangeContext truncates oversized diff patches instead of failing", () =>
1695 Effect.gen(function* () {
1696 const tmp = yield* makeTmpDir();
1697 const { initialBranch } = yield* initRepoWithCommit(tmp);
1698 const core = yield* GitCore;
1699
1700 yield* core.createBranch({ cwd: tmp, branch: "feature/large-range-context" });
1701 yield* core.checkoutBranch({ cwd: tmp, branch: "feature/large-range-context" });
1702 yield* writeTextFile(path.join(tmp, "large.txt"), buildLargeText());
1703 yield* git(tmp, ["add", "large.txt"]);
1704 yield* git(tmp, ["commit", "-m", "Add large range context"]);
1705
1706 const rangeContext = yield* core.readRangeContext(tmp, initialBranch);
1707 expect(rangeContext.commitSummary).toContain("Add large range context");
1708 expect(rangeContext.diffSummary).toContain("large.txt");
1709 expect(rangeContext.diffPatch).toContain("[truncated]");
1710 }),
1711 );
1712
1673 it.effect("pushes with upstream setup and then skips when up to date", () =>1713 it.effect("pushes with upstream setup and then skips when up to date", () =>
1674 Effect.gen(function* () {1714 Effect.gen(function* () {
1675 const tmp = yield* makeTmpDir();1715 const tmp = yield* makeTmpDir();
apps/server/src/git/Layers/GitCore.ts+76-19
...@@ -32,6 +32,11 @@ import { decodeJsonResult } from "@t3tools/shared/schemaJson";...@@ -32,6 +32,11 @@ import { decodeJsonResult } from "@t3tools/shared/schemaJson";
3232
33const DEFAULT_TIMEOUT_MS = 30_000;33const DEFAULT_TIMEOUT_MS = 30_000;
34const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000;34const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000;
35const OUTPUT_TRUNCATED_MARKER = "\n\n[truncated]";
36const PREPARED_COMMIT_PATCH_MAX_OUTPUT_BYTES = 49_000;
37const RANGE_COMMIT_SUMMARY_MAX_OUTPUT_BYTES = 19_000;
38const RANGE_DIFF_SUMMARY_MAX_OUTPUT_BYTES = 19_000;
39const RANGE_DIFF_PATCH_MAX_OUTPUT_BYTES = 59_000;
35const STATUS_UPSTREAM_REFRESH_INTERVAL = Duration.seconds(15);40const STATUS_UPSTREAM_REFRESH_INTERVAL = Duration.seconds(15);
36const STATUS_UPSTREAM_REFRESH_TIMEOUT = Duration.seconds(5);41const STATUS_UPSTREAM_REFRESH_TIMEOUT = Duration.seconds(5);
37const STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY = 2_048;42const STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY = 2_048;
...@@ -53,6 +58,8 @@ interface ExecuteGitOptions {...@@ -53,6 +58,8 @@ interface ExecuteGitOptions {
53 timeoutMs?: number | undefined;58 timeoutMs?: number | undefined;
54 allowNonZeroExit?: boolean | undefined;59 allowNonZeroExit?: boolean | undefined;
55 fallbackErrorMessage?: string | undefined;60 fallbackErrorMessage?: string | undefined;
61 maxOutputBytes?: number | undefined;
62 truncateOutputAtMaxBytes?: boolean | undefined;
56 progress?: ExecuteGitProgress | undefined;63 progress?: ExecuteGitProgress | undefined;
57}64}
5865
...@@ -439,12 +446,14 @@ const collectOutput = Effect.fn(function* <E>(...@@ -439,12 +446,14 @@ const collectOutput = Effect.fn(function* <E>(
439 input: Pick<ExecuteGitInput, "operation" | "cwd" | "args">,446 input: Pick<ExecuteGitInput, "operation" | "cwd" | "args">,
440 stream: Stream.Stream<Uint8Array, E>,447 stream: Stream.Stream<Uint8Array, E>,
441 maxOutputBytes: number,448 maxOutputBytes: number,
449 truncateOutputAtMaxBytes: boolean,
442 onLine: ((line: string) => Effect.Effect<void, never>) | undefined,450 onLine: ((line: string) => Effect.Effect<void, never>) | undefined,
443): Effect.fn.Return<string, GitCommandError> {451): Effect.fn.Return<string, GitCommandError> {
444 const decoder = new TextDecoder();452 const decoder = new TextDecoder();
445 let bytes = 0;453 let bytes = 0;
446 let text = "";454 let text = "";
447 let lineBuffer = "";455 let lineBuffer = "";
456 let truncated = false;
448457
449 const emitCompleteLines = (flush: boolean) =>458 const emitCompleteLines = (flush: boolean) =>
450 Effect.gen(function* () {459 Effect.gen(function* () {
...@@ -469,8 +478,11 @@ const collectOutput = Effect.fn(function* <E>(...@@ -469,8 +478,11 @@ const collectOutput = Effect.fn(function* <E>(
469478
470 yield* Stream.runForEach(stream, (chunk) =>479 yield* Stream.runForEach(stream, (chunk) =>
471 Effect.gen(function* () {480 Effect.gen(function* () {
472 bytes += chunk.byteLength;481 if (truncateOutputAtMaxBytes && truncated) {
473 if (bytes > maxOutputBytes) {482 return;
483 }
484 const nextBytes = bytes + chunk.byteLength;
485 if (!truncateOutputAtMaxBytes && nextBytes > maxOutputBytes) {
474 return yield* new GitCommandError({486 return yield* new GitCommandError({
475 operation: input.operation,487 operation: input.operation,
476 command: quoteGitCommand(input.args),488 command: quoteGitCommand(input.args),
...@@ -478,18 +490,26 @@ const collectOutput = Effect.fn(function* <E>(...@@ -478,18 +490,26 @@ const collectOutput = Effect.fn(function* <E>(
478 detail: `${quoteGitCommand(input.args)} output exceeded ${maxOutputBytes} bytes and was truncated.`,490 detail: `${quoteGitCommand(input.args)} output exceeded ${maxOutputBytes} bytes and was truncated.`,
479 });491 });
480 }492 }
481 const decoded = decoder.decode(chunk, { stream: true });493
494 const chunkToDecode =
495 truncateOutputAtMaxBytes && nextBytes > maxOutputBytes
496 ? chunk.subarray(0, Math.max(0, maxOutputBytes - bytes))
497 : chunk;
498 bytes += chunkToDecode.byteLength;
499 truncated = truncateOutputAtMaxBytes && nextBytes > maxOutputBytes;
500
501 const decoded = decoder.decode(chunkToDecode, { stream: !truncated });
482 text += decoded;502 text += decoded;
483 lineBuffer += decoded;503 lineBuffer += decoded;
484 yield* emitCompleteLines(false);504 yield* emitCompleteLines(false);
485 }),505 }),
486 ).pipe(Effect.mapError(toGitCommandError(input, "output stream failed.")));506 ).pipe(Effect.mapError(toGitCommandError(input, "output stream failed.")));
487507
488 const remainder = decoder.decode();508 const remainder = truncated ? "" : decoder.decode();
489 text += remainder;509 text += remainder;
490 lineBuffer += remainder;510 lineBuffer += remainder;
491 yield* emitCompleteLines(true);511 yield* emitCompleteLines(true);
492 return text;512 return truncated ? `${text}${OUTPUT_TRUNCATED_MARKER}` : text;
493});513});
494514
495export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"] }) =>515export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"] }) =>
...@@ -511,6 +531,7 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"...@@ -511,6 +531,7 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
511 } as const;531 } as const;
512 const timeoutMs = input.timeoutMs ?? DEFAULT_TIMEOUT_MS;532 const timeoutMs = input.timeoutMs ?? DEFAULT_TIMEOUT_MS;
513 const maxOutputBytes = input.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;533 const maxOutputBytes = input.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
534 const truncateOutputAtMaxBytes = input.truncateOutputAtMaxBytes ?? false;
514535
515 const commandEffect = Effect.gen(function* () {536 const commandEffect = Effect.gen(function* () {
516 const trace2Monitor = yield* createTrace2Monitor(commandInput, input.progress).pipe(537 const trace2Monitor = yield* createTrace2Monitor(commandInput, input.progress).pipe(
...@@ -537,12 +558,14 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"...@@ -537,12 +558,14 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
537 commandInput,558 commandInput,
538 child.stdout,559 child.stdout,
539 maxOutputBytes,560 maxOutputBytes,
561 truncateOutputAtMaxBytes,
540 input.progress?.onStdoutLine,562 input.progress?.onStdoutLine,
541 ),563 ),
542 collectOutput(564 collectOutput(
543 commandInput,565 commandInput,
544 child.stderr,566 child.stderr,
545 maxOutputBytes,567 maxOutputBytes,
568 truncateOutputAtMaxBytes,
546 input.progress?.onStderrLine,569 input.progress?.onStderrLine,
547 ),570 ),
548 child.exitCode.pipe(571 child.exitCode.pipe(
...@@ -603,6 +626,10 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"...@@ -603,6 +626,10 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
603 args,626 args,
604 allowNonZeroExit: true,627 allowNonZeroExit: true,
605 ...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),628 ...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
629 ...(options.maxOutputBytes !== undefined ? { maxOutputBytes: options.maxOutputBytes } : {}),
630 ...(options.truncateOutputAtMaxBytes !== undefined
631 ? { truncateOutputAtMaxBytes: options.truncateOutputAtMaxBytes }
632 : {}),
606 ...(options.progress ? { progress: options.progress } : {}),633 ...(options.progress ? { progress: options.progress } : {}),
607 }).pipe(634 }).pipe(
608 Effect.flatMap((result) => {635 Effect.flatMap((result) => {
...@@ -647,6 +674,14 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"...@@ -647,6 +674,14 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
647 Effect.map((result) => result.stdout),674 Effect.map((result) => result.stdout),
648 );675 );
649676
677 const runGitStdoutWithOptions = (
678 operation: string,
679 cwd: string,
680 args: readonly string[],
681 options: ExecuteGitOptions = {},
682 ): Effect.Effect<string, GitCommandError> =>
683 executeGit(operation, cwd, args, options).pipe(Effect.map((result) => result.stdout));
684
650 const branchExists = (cwd: string, branch: string): Effect.Effect<boolean, GitCommandError> =>685 const branchExists = (cwd: string, branch: string): Effect.Effect<boolean, GitCommandError> =>
651 executeGit(686 executeGit(
652 "GitCore.branchExists",687 "GitCore.branchExists",
...@@ -1162,12 +1197,15 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"...@@ -1162,12 +1197,15 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
1162 return null;1197 return null;
1163 }1198 }
11641199
1165 const stagedPatch = yield* runGitStdout("GitCore.prepareCommitContext.stagedPatch", cwd, [1200 const stagedPatch = yield* runGitStdoutWithOptions(
1166 "diff",1201 "GitCore.prepareCommitContext.stagedPatch",
1167 "--cached",1202 cwd,
1168 "--patch",1203 ["diff", "--cached", "--patch", "--minimal"],
1169 "--minimal",1204 {
1170 ]);1205 maxOutputBytes: PREPARED_COMMIT_PATCH_MAX_OUTPUT_BYTES,
1206 truncateOutputAtMaxBytes: true,
1207 },
1208 );
11711209
1172 return {1210 return {
1173 stagedSummary,1211 stagedSummary,
...@@ -1363,14 +1401,33 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"...@@ -1363,14 +1401,33 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
1363 const range = `${baseBranch}..HEAD`;1401 const range = `${baseBranch}..HEAD`;
1364 const [commitSummary, diffSummary, diffPatch] = yield* Effect.all(1402 const [commitSummary, diffSummary, diffPatch] = yield* Effect.all(
1365 [1403 [
1366 runGitStdout("GitCore.readRangeContext.log", cwd, ["log", "--oneline", range]),1404 runGitStdoutWithOptions(
1367 runGitStdout("GitCore.readRangeContext.diffStat", cwd, ["diff", "--stat", range]),1405 "GitCore.readRangeContext.log",
1368 runGitStdout("GitCore.readRangeContext.diffPatch", cwd, [1406 cwd,
1369 "diff",1407 ["log", "--oneline", range],
1370 "--patch",1408 {
1371 "--minimal",1409 maxOutputBytes: RANGE_COMMIT_SUMMARY_MAX_OUTPUT_BYTES,
1372 range,1410 truncateOutputAtMaxBytes: true,
1373 ]),1411 },
1412 ),
1413 runGitStdoutWithOptions(
1414 "GitCore.readRangeContext.diffStat",
1415 cwd,
1416 ["diff", "--stat", range],
1417 {
1418 maxOutputBytes: RANGE_DIFF_SUMMARY_MAX_OUTPUT_BYTES,
1419 truncateOutputAtMaxBytes: true,
1420 },
1421 ),
1422 runGitStdoutWithOptions(
1423 "GitCore.readRangeContext.diffPatch",
1424 cwd,
1425 ["diff", "--patch", "--minimal", range],
1426 {
1427 maxOutputBytes: RANGE_DIFF_PATCH_MAX_OUTPUT_BYTES,
1428 truncateOutputAtMaxBytes: true,
1429 },
1430 ),
1374 ],1431 ],
1375 { concurrency: "unbounded" },1432 { concurrency: "unbounded" },
1376 );1433 );
apps/server/src/git/Services/GitCore.ts+1
...@@ -32,6 +32,7 @@ export interface ExecuteGitInput {...@@ -32,6 +32,7 @@ export interface ExecuteGitInput {
32 readonly allowNonZeroExit?: boolean;32 readonly allowNonZeroExit?: boolean;
33 readonly timeoutMs?: number;33 readonly timeoutMs?: number;
34 readonly maxOutputBytes?: number;34 readonly maxOutputBytes?: number;
35 readonly truncateOutputAtMaxBytes?: boolean;
35 readonly progress?: ExecuteGitProgress;36 readonly progress?: ExecuteGitProgress;
36}37}
3738