authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-03-28 20:50:23-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-03-28 20:50:23-07:00
log9e60597174dcd69ae405d3966a374213c745388a
tree45c2b4470177339298bfb8387006b64f7fe8a87d
parentafc807ac3f8cc827d5f30f587a599dd077f244ff
signature Signed by PGP key B5690EEEBB952194

Truncate oversized git diffs instead of failing (#1499)

4 files changed, 239 insertions(+), 19 deletions(-)

apps/server/src/checkpointing/Layers/CheckpointStore.test.ts created+122
......@@ -0,0 +1,122 @@
1import path from "node:path";
2
3import * as NodeServices from "@effect/platform-node/NodeServices";
4import { it } from "@effect/vitest";
5import { Effect, FileSystem, Layer, PlatformError, Scope } from "effect";
6import { describe, expect } from "vitest";
7
8import { checkpointRefForThreadTurn } from "../Utils.ts";
9import { CheckpointStoreLive } from "./CheckpointStore.ts";
10import { CheckpointStore } from "../Services/CheckpointStore.ts";
11import { GitCoreLive } from "../../git/Layers/GitCore.ts";
12import { GitCore } from "../../git/Services/GitCore.ts";
13import { GitCommandError } from "../../git/Errors.ts";
14import { ServerConfig } from "../../config.ts";
15import { ThreadId } from "@t3tools/contracts";
16
17const ServerConfigLayer = ServerConfig.layerTest(process.cwd(), {
18 prefix: "t3-checkpoint-store-test-",
19});
20const GitCoreTestLayer = GitCoreLive.pipe(
21 Layer.provide(ServerConfigLayer),
22 Layer.provide(NodeServices.layer),
23);
24const CheckpointStoreTestLayer = CheckpointStoreLive.pipe(
25 Layer.provide(GitCoreTestLayer),
26 Layer.provide(NodeServices.layer),
27);
28const TestLayer = Layer.mergeAll(NodeServices.layer, GitCoreTestLayer, CheckpointStoreTestLayer);
29
30function makeTmpDir(
31 prefix = "checkpoint-store-test-",
32): Effect.Effect<string, PlatformError.PlatformError, FileSystem.FileSystem | Scope.Scope> {
33 return Effect.gen(function* () {
34 const fileSystem = yield* FileSystem.FileSystem;
35 return yield* fileSystem.makeTempDirectoryScoped({ prefix });
36 });
37}
38
39function writeTextFile(
40 filePath: string,
41 contents: string,
42): Effect.Effect<void, PlatformError.PlatformError, FileSystem.FileSystem> {
43 return Effect.gen(function* () {
44 const fileSystem = yield* FileSystem.FileSystem;
45 yield* fileSystem.writeFileString(filePath, contents);
46 });
47}
48
49function git(
50 cwd: string,
51 args: ReadonlyArray<string>,
52): Effect.Effect<string, GitCommandError, GitCore> {
53 return Effect.gen(function* () {
54 const gitCore = yield* GitCore;
55 const result = yield* gitCore.execute({
56 operation: "CheckpointStore.test.git",
57 cwd,
58 args,
59 timeoutMs: 10_000,
60 });
61 return result.stdout.trim();
62 });
63}
64
65function initRepoWithCommit(
66 cwd: string,
67): Effect.Effect<
68 void,
69 GitCommandError | PlatformError.PlatformError,
70 GitCore | FileSystem.FileSystem
71> {
72 return Effect.gen(function* () {
73 const core = yield* GitCore;
74 yield* core.initRepo({ cwd });
75 yield* git(cwd, ["config", "user.email", "test@test.com"]);
76 yield* git(cwd, ["config", "user.name", "Test"]);
77 yield* writeTextFile(path.join(cwd, "README.md"), "# test\n");
78 yield* git(cwd, ["add", "."]);
79 yield* git(cwd, ["commit", "-m", "initial commit"]);
80 });
81}
82
83function buildLargeText(lineCount = 20_000): string {
84 return Array.from({ length: lineCount }, (_, index) => `line ${String(index).padStart(5, "0")}`)
85 .join("\n")
86 .concat("\n");
87}
88
89it.layer(TestLayer)("CheckpointStoreLive", (it) => {
90 describe("diffCheckpoints", () => {
91 it.effect("returns full oversized checkpoint diffs without truncation", () =>
92 Effect.gen(function* () {
93 const tmp = yield* makeTmpDir();
94 yield* initRepoWithCommit(tmp);
95 const checkpointStore = yield* CheckpointStore;
96 const threadId = ThreadId.makeUnsafe("thread-checkpoint-store");
97 const fromCheckpointRef = checkpointRefForThreadTurn(threadId, 0);
98 const toCheckpointRef = checkpointRefForThreadTurn(threadId, 1);
99
100 yield* checkpointStore.captureCheckpoint({
101 cwd: tmp,
102 checkpointRef: fromCheckpointRef,
103 });
104 yield* writeTextFile(path.join(tmp, "README.md"), buildLargeText());
105 yield* checkpointStore.captureCheckpoint({
106 cwd: tmp,
107 checkpointRef: toCheckpointRef,
108 });
109
110 const diff = yield* checkpointStore.diffCheckpoints({
111 cwd: tmp,
112 fromCheckpointRef,
113 toCheckpointRef,
114 });
115
116 expect(diff).toContain("diff --git");
117 expect(diff).not.toContain("[truncated]");
118 expect(diff).toContain("+line 19999");
119 }),
120 );
121 });
122});
apps/server/src/git/Layers/GitCore.test.ts+40
......@@ -132,6 +132,12 @@ function commitWithDate(
132132 });
133133}
134134
135function buildLargeText(lineCount = 20_000): string {
136 return Array.from({ length: lineCount }, (_, index) => `line ${String(index).padStart(5, "0")}`)
137 .join("\n")
138 .concat("\n");
139}
140
135141// ── Tests ──
136142
137143it.layer(TestLayer)("git integration", (it) => {
......@@ -1670,6 +1676,40 @@ it.layer(TestLayer)("git integration", (it) => {
16701676 }),
16711677 );
16721678
1679 it.effect("prepareCommitContext truncates oversized staged patches instead of failing", () =>
1680 Effect.gen(function* () {
1681 const tmp = yield* makeTmpDir();
1682 yield* initRepoWithCommit(tmp);
1683 const core = yield* GitCore;
1684
1685 yield* writeTextFile(path.join(tmp, "README.md"), buildLargeText());
1686
1687 const context = yield* core.prepareCommitContext(tmp);
1688 expect(context).not.toBeNull();
1689 expect(context!.stagedSummary).toContain("README.md");
1690 expect(context!.stagedPatch).toContain("[truncated]");
1691 }),
1692 );
1693
1694 it.effect("readRangeContext truncates oversized diff patches instead of failing", () =>
1695 Effect.gen(function* () {
1696 const tmp = yield* makeTmpDir();
1697 const { initialBranch } = yield* initRepoWithCommit(tmp);
1698 const core = yield* GitCore;
1699
1700 yield* core.createBranch({ cwd: tmp, branch: "feature/large-range-context" });
1701 yield* core.checkoutBranch({ cwd: tmp, branch: "feature/large-range-context" });
1702 yield* writeTextFile(path.join(tmp, "large.txt"), buildLargeText());
1703 yield* git(tmp, ["add", "large.txt"]);
1704 yield* git(tmp, ["commit", "-m", "Add large range context"]);
1705
1706 const rangeContext = yield* core.readRangeContext(tmp, initialBranch);
1707 expect(rangeContext.commitSummary).toContain("Add large range context");
1708 expect(rangeContext.diffSummary).toContain("large.txt");
1709 expect(rangeContext.diffPatch).toContain("[truncated]");
1710 }),
1711 );
1712
16731713 it.effect("pushes with upstream setup and then skips when up to date", () =>
16741714 Effect.gen(function* () {
16751715 const tmp = yield* makeTmpDir();
apps/server/src/git/Layers/GitCore.ts+76-19
......@@ -32,6 +32,11 @@ import { decodeJsonResult } from "@t3tools/shared/schemaJson";
3232
3333const DEFAULT_TIMEOUT_MS = 30_000;
3434const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000;
35const OUTPUT_TRUNCATED_MARKER = "\n\n[truncated]";
36const PREPARED_COMMIT_PATCH_MAX_OUTPUT_BYTES = 49_000;
37const RANGE_COMMIT_SUMMARY_MAX_OUTPUT_BYTES = 19_000;
38const RANGE_DIFF_SUMMARY_MAX_OUTPUT_BYTES = 19_000;
39const RANGE_DIFF_PATCH_MAX_OUTPUT_BYTES = 59_000;
3540const STATUS_UPSTREAM_REFRESH_INTERVAL = Duration.seconds(15);
3641const STATUS_UPSTREAM_REFRESH_TIMEOUT = Duration.seconds(5);
3742const STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY = 2_048;
......@@ -53,6 +58,8 @@ interface ExecuteGitOptions {
5358 timeoutMs?: number | undefined;
5459 allowNonZeroExit?: boolean | undefined;
5560 fallbackErrorMessage?: string | undefined;
61 maxOutputBytes?: number | undefined;
62 truncateOutputAtMaxBytes?: boolean | undefined;
5663 progress?: ExecuteGitProgress | undefined;
5764}
5865
......@@ -439,12 +446,14 @@ const collectOutput = Effect.fn(function* <E>(
439446 input: Pick<ExecuteGitInput, "operation" | "cwd" | "args">,
440447 stream: Stream.Stream<Uint8Array, E>,
441448 maxOutputBytes: number,
449 truncateOutputAtMaxBytes: boolean,
442450 onLine: ((line: string) => Effect.Effect<void, never>) | undefined,
443451): Effect.fn.Return<string, GitCommandError> {
444452 const decoder = new TextDecoder();
445453 let bytes = 0;
446454 let text = "";
447455 let lineBuffer = "";
456 let truncated = false;
448457
449458 const emitCompleteLines = (flush: boolean) =>
450459 Effect.gen(function* () {
......@@ -469,8 +478,11 @@ const collectOutput = Effect.fn(function* <E>(
469478
470479 yield* Stream.runForEach(stream, (chunk) =>
471480 Effect.gen(function* () {
472 bytes += chunk.byteLength;
473 if (bytes > maxOutputBytes) {
481 if (truncateOutputAtMaxBytes && truncated) {
482 return;
483 }
484 const nextBytes = bytes + chunk.byteLength;
485 if (!truncateOutputAtMaxBytes && nextBytes > maxOutputBytes) {
474486 return yield* new GitCommandError({
475487 operation: input.operation,
476488 command: quoteGitCommand(input.args),
......@@ -478,18 +490,26 @@ const collectOutput = Effect.fn(function* <E>(
478490 detail: `${quoteGitCommand(input.args)} output exceeded ${maxOutputBytes} bytes and was truncated.`,
479491 });
480492 }
481 const decoded = decoder.decode(chunk, { stream: true });
493
494 const chunkToDecode =
495 truncateOutputAtMaxBytes && nextBytes > maxOutputBytes
496 ? chunk.subarray(0, Math.max(0, maxOutputBytes - bytes))
497 : chunk;
498 bytes += chunkToDecode.byteLength;
499 truncated = truncateOutputAtMaxBytes && nextBytes > maxOutputBytes;
500
501 const decoded = decoder.decode(chunkToDecode, { stream: !truncated });
482502 text += decoded;
483503 lineBuffer += decoded;
484504 yield* emitCompleteLines(false);
485505 }),
486506 ).pipe(Effect.mapError(toGitCommandError(input, "output stream failed.")));
487507
488 const remainder = decoder.decode();
508 const remainder = truncated ? "" : decoder.decode();
489509 text += remainder;
490510 lineBuffer += remainder;
491511 yield* emitCompleteLines(true);
492 return text;
512 return truncated ? `${text}${OUTPUT_TRUNCATED_MARKER}` : text;
493513});
494514
495515export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"] }) =>
......@@ -511,6 +531,7 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
511531 } as const;
512532 const timeoutMs = input.timeoutMs ?? DEFAULT_TIMEOUT_MS;
513533 const maxOutputBytes = input.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
534 const truncateOutputAtMaxBytes = input.truncateOutputAtMaxBytes ?? false;
514535
515536 const commandEffect = Effect.gen(function* () {
516537 const trace2Monitor = yield* createTrace2Monitor(commandInput, input.progress).pipe(
......@@ -537,12 +558,14 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
537558 commandInput,
538559 child.stdout,
539560 maxOutputBytes,
561 truncateOutputAtMaxBytes,
540562 input.progress?.onStdoutLine,
541563 ),
542564 collectOutput(
543565 commandInput,
544566 child.stderr,
545567 maxOutputBytes,
568 truncateOutputAtMaxBytes,
546569 input.progress?.onStderrLine,
547570 ),
548571 child.exitCode.pipe(
......@@ -603,6 +626,10 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
603626 args,
604627 allowNonZeroExit: true,
605628 ...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
629 ...(options.maxOutputBytes !== undefined ? { maxOutputBytes: options.maxOutputBytes } : {}),
630 ...(options.truncateOutputAtMaxBytes !== undefined
631 ? { truncateOutputAtMaxBytes: options.truncateOutputAtMaxBytes }
632 : {}),
606633 ...(options.progress ? { progress: options.progress } : {}),
607634 }).pipe(
608635 Effect.flatMap((result) => {
......@@ -647,6 +674,14 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
647674 Effect.map((result) => result.stdout),
648675 );
649676
677 const runGitStdoutWithOptions = (
678 operation: string,
679 cwd: string,
680 args: readonly string[],
681 options: ExecuteGitOptions = {},
682 ): Effect.Effect<string, GitCommandError> =>
683 executeGit(operation, cwd, args, options).pipe(Effect.map((result) => result.stdout));
684
650685 const branchExists = (cwd: string, branch: string): Effect.Effect<boolean, GitCommandError> =>
651686 executeGit(
652687 "GitCore.branchExists",
......@@ -1162,12 +1197,15 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
11621197 return null;
11631198 }
11641199
1165 const stagedPatch = yield* runGitStdout("GitCore.prepareCommitContext.stagedPatch", cwd, [
1166 "diff",
1167 "--cached",
1168 "--patch",
1169 "--minimal",
1170 ]);
1200 const stagedPatch = yield* runGitStdoutWithOptions(
1201 "GitCore.prepareCommitContext.stagedPatch",
1202 cwd,
1203 ["diff", "--cached", "--patch", "--minimal"],
1204 {
1205 maxOutputBytes: PREPARED_COMMIT_PATCH_MAX_OUTPUT_BYTES,
1206 truncateOutputAtMaxBytes: true,
1207 },
1208 );
11711209
11721210 return {
11731211 stagedSummary,
......@@ -1363,14 +1401,33 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
13631401 const range = `${baseBranch}..HEAD`;
13641402 const [commitSummary, diffSummary, diffPatch] = yield* Effect.all(
13651403 [
1366 runGitStdout("GitCore.readRangeContext.log", cwd, ["log", "--oneline", range]),
1367 runGitStdout("GitCore.readRangeContext.diffStat", cwd, ["diff", "--stat", range]),
1368 runGitStdout("GitCore.readRangeContext.diffPatch", cwd, [
1369 "diff",
1370 "--patch",
1371 "--minimal",
1372 range,
1373 ]),
1404 runGitStdoutWithOptions(
1405 "GitCore.readRangeContext.log",
1406 cwd,
1407 ["log", "--oneline", range],
1408 {
1409 maxOutputBytes: RANGE_COMMIT_SUMMARY_MAX_OUTPUT_BYTES,
1410 truncateOutputAtMaxBytes: true,
1411 },
1412 ),
1413 runGitStdoutWithOptions(
1414 "GitCore.readRangeContext.diffStat",
1415 cwd,
1416 ["diff", "--stat", range],
1417 {
1418 maxOutputBytes: RANGE_DIFF_SUMMARY_MAX_OUTPUT_BYTES,
1419 truncateOutputAtMaxBytes: true,
1420 },
1421 ),
1422 runGitStdoutWithOptions(
1423 "GitCore.readRangeContext.diffPatch",
1424 cwd,
1425 ["diff", "--patch", "--minimal", range],
1426 {
1427 maxOutputBytes: RANGE_DIFF_PATCH_MAX_OUTPUT_BYTES,
1428 truncateOutputAtMaxBytes: true,
1429 },
1430 ),
13741431 ],
13751432 { concurrency: "unbounded" },
13761433 );
apps/server/src/git/Services/GitCore.ts+1
......@@ -32,6 +32,7 @@ export interface ExecuteGitInput {
3232 readonly allowNonZeroExit?: boolean;
3333 readonly timeoutMs?: number;
3434 readonly maxOutputBytes?: number;
35 readonly truncateOutputAtMaxBytes?: boolean;
3536 readonly progress?: ExecuteGitProgress;
3637}
3738