1import * as NodeServices from "@effect/platform-node/NodeServices";
2import { it } from "@effect/vitest";
3import { ClaudeSettings, ProviderInstanceId } from "@t3tools/contracts";
4import * as HostProcess from "@t3tools/shared/HostProcess";
5import { createModelSelection } from "@t3tools/shared/model";
6import * as Effect from "effect/Effect";
7import * as FileSystem from "effect/FileSystem";
8import * as Layer from "effect/Layer";
9import * as Path from "effect/Path";
10import * as Schema from "effect/Schema";
11import { expect } from "vite-plus/test";
12
13import * as ServerConfig from "../config.ts";
14import {
15 SYNTHETIC_CLAUDE_CAPABLE_MODEL,
16 SYNTHETIC_CLAUDE_COLLIDING_ALIAS,
17 SYNTHETIC_CLAUDE_MODEL_CATALOG,
18 SYNTHETIC_CLAUDE_STANDARD_MODEL,
19 SYNTHETIC_CLAUDE_THINKING_MODEL,
20} from "../provider/ClaudeModelCatalog.testFixtures.ts";
21import * as TextGeneration from "./TextGeneration.ts";
22import { sanitizeThreadTitle } from "@t3tools/provider-core/server/textGenerationUtils";
23import { makeClaudeTextGeneration } from "./ClaudeTextGeneration.ts";
24import { writeFakeCli } from "@t3tools/provider-testing/fakeCli";
25const decodeClaudeSettings = Schema.decodeSync(ClaudeSettings);
26
27const layerClaudeTextGenerationTest = ServerConfig.ServerConfig.layerTest(process.cwd(), {
28 prefix: "t3code-claude-text-generation-test-",
29}).pipe(Layer.provideMerge(NodeServices.layer));
30
31// The stub behaviour lives in Node so the same implementation runs on Windows,
32// where a shebang file is not executable and would fall through to the real
33// Claude CLI on PATH; `writeFakeCli` picks the launcher shape per host.
34function makeFakeClaudeBinary(dir: string) {
35 return Effect.gen(function* () {
36 const path = yield* Path.Path;
37 const fs = yield* FileSystem.FileSystem;
38 const platform = yield* HostProcess.Platform;
39 const binDir = path.join(dir, "bin");
40 const fixturePath = yield* path.fromFileUrl(
41 new URL("./testing/ClaudeTextGeneration.fixture.mjs", import.meta.url),
42 );
43 const source = yield* fs.readFileString(fixturePath);
44 writeFakeCli({
45 directory: binDir,
46 name: "claude",
47 platform,
48 source,
49 });
50 return binDir;
51 });
52}
53
54function withFakeClaudeEnv<A, E, R>(
55 input: {
56 output: string;
57 exitCode?: number;
58 stderr?: string;
59 argsMustContain?: string;
60 argsMustNotContain?: string;
61 stdinMustContain?: string;
62 configDirMustBe?: string;
63 cwdMustNotBe?: string;
64 claudeConfig?: Partial<ClaudeSettings>;
65 },
66 effectFn: (textGeneration: TextGeneration.TextGeneration["Service"]) => Effect.Effect<A, E, R>,
67) {
68 return Effect.gen(function* () {
69 const fs = yield* FileSystem.FileSystem;
70 const tempDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3code-claude-text-" });
71 const binDir = yield* makeFakeClaudeBinary(tempDir);
72 const pathDelimiter = (yield* HostProcess.isWindows) ? ";" : ":";
73 const previousPath = process.env.PATH;
74 const previousOutput = process.env.T3_FAKE_CLAUDE_OUTPUT;
75 const previousExitCode = process.env.T3_FAKE_CLAUDE_EXIT_CODE;
76 const previousStderr = process.env.T3_FAKE_CLAUDE_STDERR;
77 const previousArgsMustContain = process.env.T3_FAKE_CLAUDE_ARGS_MUST_CONTAIN;
78 const previousArgsMustNotContain = process.env.T3_FAKE_CLAUDE_ARGS_MUST_NOT_CONTAIN;
79 const previousStdinMustContain = process.env.T3_FAKE_CLAUDE_STDIN_MUST_CONTAIN;
80 const previousConfigDirMustBe = process.env.T3_FAKE_CLAUDE_CONFIG_DIR_MUST_BE;
81 const previousCwdMustNotBe = process.env.T3_FAKE_CLAUDE_CWD_MUST_NOT_BE;
82
83 yield* Effect.acquireRelease(
84 Effect.sync(() => {
85 process.env.PATH = `${binDir}${pathDelimiter}${previousPath ?? ""}`;
86 process.env.T3_FAKE_CLAUDE_OUTPUT = input.output;
87
88 if (input.exitCode !== undefined) {
89 process.env.T3_FAKE_CLAUDE_EXIT_CODE = String(input.exitCode);
90 } else {
91 delete process.env.T3_FAKE_CLAUDE_EXIT_CODE;
92 }
93
94 if (input.stderr !== undefined) {
95 process.env.T3_FAKE_CLAUDE_STDERR = input.stderr;
96 } else {
97 delete process.env.T3_FAKE_CLAUDE_STDERR;
98 }
99
100 if (input.argsMustContain !== undefined) {
101 process.env.T3_FAKE_CLAUDE_ARGS_MUST_CONTAIN = input.argsMustContain;
102 } else {
103 delete process.env.T3_FAKE_CLAUDE_ARGS_MUST_CONTAIN;
104 }
105
106 if (input.argsMustNotContain !== undefined) {
107 process.env.T3_FAKE_CLAUDE_ARGS_MUST_NOT_CONTAIN = input.argsMustNotContain;
108 } else {
109 delete process.env.T3_FAKE_CLAUDE_ARGS_MUST_NOT_CONTAIN;
110 }
111
112 if (input.stdinMustContain !== undefined) {
113 process.env.T3_FAKE_CLAUDE_STDIN_MUST_CONTAIN = input.stdinMustContain;
114 } else {
115 delete process.env.T3_FAKE_CLAUDE_STDIN_MUST_CONTAIN;
116 }
117
118 if (input.cwdMustNotBe !== undefined) {
119 process.env.T3_FAKE_CLAUDE_CWD_MUST_NOT_BE = input.cwdMustNotBe;
120 } else {
121 delete process.env.T3_FAKE_CLAUDE_CWD_MUST_NOT_BE;
122 }
123
124 if (input.configDirMustBe !== undefined) {
125 process.env.T3_FAKE_CLAUDE_CONFIG_DIR_MUST_BE = input.configDirMustBe;
126 } else {
127 delete process.env.T3_FAKE_CLAUDE_CONFIG_DIR_MUST_BE;
128 }
129 }),
130 () =>
131 Effect.sync(() => {
132 process.env.PATH = previousPath;
133
134 if (previousOutput === undefined) {
135 delete process.env.T3_FAKE_CLAUDE_OUTPUT;
136 } else {
137 process.env.T3_FAKE_CLAUDE_OUTPUT = previousOutput;
138 }
139
140 if (previousExitCode === undefined) {
141 delete process.env.T3_FAKE_CLAUDE_EXIT_CODE;
142 } else {
143 process.env.T3_FAKE_CLAUDE_EXIT_CODE = previousExitCode;
144 }
145
146 if (previousStderr === undefined) {
147 delete process.env.T3_FAKE_CLAUDE_STDERR;
148 } else {
149 process.env.T3_FAKE_CLAUDE_STDERR = previousStderr;
150 }
151
152 if (previousArgsMustContain === undefined) {
153 delete process.env.T3_FAKE_CLAUDE_ARGS_MUST_CONTAIN;
154 } else {
155 process.env.T3_FAKE_CLAUDE_ARGS_MUST_CONTAIN = previousArgsMustContain;
156 }
157
158 if (previousArgsMustNotContain === undefined) {
159 delete process.env.T3_FAKE_CLAUDE_ARGS_MUST_NOT_CONTAIN;
160 } else {
161 process.env.T3_FAKE_CLAUDE_ARGS_MUST_NOT_CONTAIN = previousArgsMustNotContain;
162 }
163
164 if (previousStdinMustContain === undefined) {
165 delete process.env.T3_FAKE_CLAUDE_STDIN_MUST_CONTAIN;
166 } else {
167 process.env.T3_FAKE_CLAUDE_STDIN_MUST_CONTAIN = previousStdinMustContain;
168 }
169
170 if (previousCwdMustNotBe === undefined) {
171 delete process.env.T3_FAKE_CLAUDE_CWD_MUST_NOT_BE;
172 } else {
173 process.env.T3_FAKE_CLAUDE_CWD_MUST_NOT_BE = previousCwdMustNotBe;
174 }
175
176 if (previousConfigDirMustBe === undefined) {
177 delete process.env.T3_FAKE_CLAUDE_CONFIG_DIR_MUST_BE;
178 } else {
179 process.env.T3_FAKE_CLAUDE_CONFIG_DIR_MUST_BE = previousConfigDirMustBe;
180 }
181 }),
182 );
183
184 const config = decodeClaudeSettings(input.claudeConfig ?? {});
185 const textGeneration = yield* makeClaudeTextGeneration(
186 config,
187 undefined,
188 Effect.succeed(SYNTHETIC_CLAUDE_MODEL_CATALOG),
189 );
190 return yield* effectFn(textGeneration);
191 }).pipe(Effect.scoped);
192}
193
194it.layer(layerClaudeTextGenerationTest)("ClaudeTextGeneration", (it) => {
195 it.effect("forwards Claude thinking settings without passing unsupported effort", () =>
196 withFakeClaudeEnv(
197 {
198 output: JSON.stringify({
199 structured_output: {
200 subject: "Add important change",
201 body: "",
202 },
203 }),
204 argsMustContain: '--settings {"disableAllHooks":true,"alwaysThinkingEnabled":false}',
205 argsMustNotContain: "--effort",
206 },
207 (textGeneration) =>
208 Effect.gen(function* () {
209 const generated = yield* textGeneration.generateCommitMessage({
210 cwd: process.cwd(),
211 branch: "feature/claude-effect",
212 stagedSummary: "M README.md",
213 stagedPatch: "diff --git a/README.md b/README.md",
214 modelSelection: {
215 ...createModelSelection(
216 ProviderInstanceId.make("claudeAgent"),
217 SYNTHETIC_CLAUDE_THINKING_MODEL,
218 [
219 { id: "thinking", value: false },
220 { id: "effort", value: "high" },
221 ],
222 ),
223 },
224 });
225
226 expect(generated.subject).toBe("Add important change");
227 }),
228 ),
229 );
230
231 it.effect("keeps a configured custom alias opaque to the Claude CLI", () =>
232 withFakeClaudeEnv(
233 {
234 output: JSON.stringify({
235 structured_output: {
236 title: "Keep custom model",
237 body: "",
238 },
239 }),
240 argsMustContain: `--model ${SYNTHETIC_CLAUDE_COLLIDING_ALIAS} --settings`,
241 claudeConfig: { customModels: [SYNTHETIC_CLAUDE_COLLIDING_ALIAS] },
242 },
243 (textGeneration) =>
244 Effect.gen(function* () {
245 const generated = yield* textGeneration.generatePrContent({
246 cwd: process.cwd(),
247 baseBranch: "main",
248 headBranch: "feature/custom-model",
249 commitSummary: "Keep custom model",
250 diffSummary: "1 file changed",
251 diffPatch: "diff --git a/README.md b/README.md",
252 modelSelection: createModelSelection(
253 ProviderInstanceId.make("claudeAgent"),
254 SYNTHETIC_CLAUDE_COLLIDING_ALIAS,
255 [
256 { id: "effort", value: "max" },
257 { id: "fastMode", value: true },
258 { id: "contextWindow", value: "expanded" },
259 ],
260 ),
261 });
262
263 expect(generated.title).toBe("Keep custom model");
264 }),
265 ),
266 );
267
268 it.effect(
269 "keeps canonical built-in capabilities when a custom model collides with its alias",
270 () =>
271 withFakeClaudeEnv(
272 {
273 output: JSON.stringify({
274 structured_output: {
275 title: "Improve orchestration flow",
276 body: "Body",
277 },
278 }),
279 argsMustContain: `--model ${SYNTHETIC_CLAUDE_CAPABLE_MODEL}[expanded] --effort max --settings {"disableAllHooks":true,"fastMode":true}`,
280 claudeConfig: { customModels: [SYNTHETIC_CLAUDE_COLLIDING_ALIAS] },
281 },
282 (textGeneration) =>
283 Effect.gen(function* () {
284 const generated = yield* textGeneration.generatePrContent({
285 cwd: process.cwd(),
286 baseBranch: "main",
287 headBranch: "feature/claude-effect",
288 commitSummary: "Improve orchestration",
289 diffSummary: "1 file changed",
290 diffPatch: "diff --git a/README.md b/README.md",
291 modelSelection: {
292 ...createModelSelection(
293 ProviderInstanceId.make("claudeAgent"),
294 SYNTHETIC_CLAUDE_CAPABLE_MODEL,
295 [
296 { id: "effort", value: "max" },
297 { id: "fastMode", value: true },
298 ],
299 ),
300 },
301 });
302
303 expect(generated.title).toBe("Improve orchestration flow");
304 }),
305 ),
306 );
307
308 it.effect(
309 "generates thread titles outside the project with tools, skills, and hooks disabled",
310 () =>
311 withFakeClaudeEnv(
312 {
313 output: JSON.stringify({
314 structured_output: {
315 title:
316 ' "Reconnect failures after restart because the session state does not recover" ',
317 },
318 }),
319 cwdMustNotBe: process.cwd(),
320 stdinMustContain: "/call-script",
321 },
322 (textGeneration) =>
323 Effect.gen(function* () {
324 const generated = yield* textGeneration.generateThreadTitle({
325 cwd: process.cwd(),
326 message: "/call-script",
327 modelSelection: {
328 instanceId: ProviderInstanceId.make("claudeAgent"),
329 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
330 },
331 });
332
333 expect(generated.title).toBe(
334 sanitizeThreadTitle(
335 '"Reconnect failures after restart because the session state does not recover"',
336 ),
337 );
338 }),
339 ),
340 );
341
342 it.effect("generates branch names from skill prompts without executable capabilities", () =>
343 withFakeClaudeEnv(
344 {
345 output: JSON.stringify({ structured_output: { branch: "call-script" } }),
346 stdinMustContain: "/call-script",
347 },
348 (textGeneration) =>
349 Effect.gen(function* () {
350 const generated = yield* textGeneration.generateBranchName({
351 cwd: process.cwd(),
352 message: "/call-script",
353 modelSelection: {
354 instanceId: ProviderInstanceId.make("claudeAgent"),
355 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
356 },
357 });
358
359 expect(generated.branch).toBe("call-script");
360 }),
361 ),
362 );
363
364 it.effect("runs Claude text generation with the configured CLAUDE_CONFIG_DIR", () =>
365 Effect.gen(function* () {
366 const path = yield* Path.Path;
367 const claudeConfigDir = path.join(process.cwd(), ".claude-work-test");
368 return yield* withFakeClaudeEnv(
369 {
370 output: JSON.stringify({
371 structured_output: {
372 title: "Use Claude home",
373 },
374 }),
375 configDirMustBe: claudeConfigDir,
376 claudeConfig: { homePath: claudeConfigDir },
377 },
378 (textGeneration) =>
379 Effect.gen(function* () {
380 const generated = yield* textGeneration.generateThreadTitle({
381 cwd: process.cwd(),
382 message: "thread title",
383 modelSelection: {
384 instanceId: ProviderInstanceId.make("claudeAgent"),
385 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
386 },
387 });
388
389 expect(generated.title).toBe(sanitizeThreadTitle("Use Claude home"));
390 }),
391 );
392 }),
393 );
394
395 it.effect.each([
396 { verbose: false, label: "normal" },
397 { verbose: true, label: "verbose" },
398 ])("unwraps a JSON title in $label Claude output", ({ verbose }) => {
399 const result = {
400 type: "result",
401 structured_output: { title: '{"title": "Refresh ev-stg APP ASG instances"}' },
402 };
403 return withFakeClaudeEnv(
404 { output: JSON.stringify(verbose ? [result] : result) },
405 (textGeneration) =>
406 Effect.gen(function* () {
407 const generated = yield* textGeneration.generateThreadTitle({
408 cwd: process.cwd(),
409 message: "Refresh ev-stg APP ASG instances",
410 modelSelection: {
411 instanceId: ProviderInstanceId.make("claudeAgent"),
412 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
413 },
414 });
415
416 expect(generated.title).toBe("Refresh ev-stg APP ASG instances");
417 }),
418 );
419 });
420
421 it.effect.each([
422 { previousTitle: undefined, label: "generating" },
423 { previousTitle: "Old thread title", label: "regenerating" },
424 ])("reads the result from verbose Claude output when $label a title", ({ previousTitle }) =>
425 withFakeClaudeEnv(
426 {
427 output: JSON.stringify([
428 { type: "system", subtype: "init" },
429 { type: "assistant", message: { content: [] } },
430 { type: "user", message: { content: [] } },
431 { type: "rate_limit_event" },
432 {
433 type: "result",
434 subtype: "success",
435 result: '{"title":"Refresh ev-stg APP ASG Instances"}',
436 structured_output: { title: "Refresh ev-stg APP ASG Instances" },
437 },
438 ]),
439 },
440 (textGeneration) =>
441 Effect.gen(function* () {
442 const generated = yield* textGeneration.generateThreadTitle({
443 cwd: process.cwd(),
444 message: "Refresh ev-stg APP ASG instances",
445 previousTitle,
446 modelSelection: {
447 instanceId: ProviderInstanceId.make("claudeAgent"),
448 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
449 },
450 });
451
452 expect(generated.title).toBe("Refresh ev-stg APP ASG Instances");
453 }),
454 ),
455 );
456
457 it.effect.each([
458 ["empty message array", []],
459 ["missing result", [{ type: "assistant", structured_output: { title: "Not a result" } }]],
460 ["invalid title", [{ type: "result", structured_output: { title: 42 } }]],
461 [
462 "final result without structured output",
463 [
464 { type: "result", structured_output: { title: "Earlier result" } },
465 { type: "result", subtype: "error_max_structured_output_retries" },
466 ],
467 ],
468 ] as const)("rejects verbose Claude output with %s", ([, output]) =>
469 withFakeClaudeEnv({ output: JSON.stringify(output) }, (textGeneration) =>
470 Effect.gen(function* () {
471 const error = yield* Effect.flip(
472 textGeneration.generateThreadTitle({
473 cwd: process.cwd(),
474 message: "Name this thread",
475 modelSelection: {
476 instanceId: ProviderInstanceId.make("claudeAgent"),
477 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
478 },
479 }),
480 );
481
482 expect(error._tag).toBe("TextGenerationError");
483 }),
484 ),
485 );
486
487 const apiError = "API Error: 400 Model blocked by guardrail: 4 endpoints excluded";
488 const wrapperNotice = "Using the OpenRouter credential from the global credential file.";
489 it.effect.each([
490 [
491 "the stdout error over a stderr notice",
492 JSON.stringify({ is_error: true, api_error_status: 400, result: apiError }),
493 apiError,
494 ],
495 [
496 "the verbose stdout error over a stderr notice",
497 JSON.stringify([
498 { type: "system", subtype: "init" },
499 { type: "result", is_error: true, result: apiError },
500 { type: "result", is_error: true, result: " " },
501 ]),
502 apiError,
503 ],
504 ["stderr when stdout has no error", "not json", wrapperNotice],
505 ] as const)("reports %s when the Claude CLI fails", ([, output, expected]) =>
506 withFakeClaudeEnv({ output, stderr: wrapperNotice, exitCode: 1 }, (textGeneration) =>
507 Effect.gen(function* () {
508 const error = yield* Effect.flip(
509 textGeneration.generateThreadTitle({
510 cwd: process.cwd(),
511 message: "Name this thread",
512 modelSelection: {
513 instanceId: ProviderInstanceId.make("claudeAgent"),
514 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
515 },
516 }),
517 );
518
519 expect(error.detail).toBe(`Claude CLI command failed: ${expected}`);
520 }),
521 ),
522 );
523
524 it.effect("falls back when Claude thread title normalization becomes whitespace-only", () =>
525 withFakeClaudeEnv(
526 {
527 output: JSON.stringify({
528 structured_output: {
529 title: ' """ """ ',
530 },
531 }),
532 },
533 (textGeneration) =>
534 Effect.gen(function* () {
535 const generated = yield* textGeneration.generateThreadTitle({
536 cwd: process.cwd(),
537 message: "Name this thread.",
538 modelSelection: {
539 instanceId: ProviderInstanceId.make("claudeAgent"),
540 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
541 },
542 });
543
544 expect(generated.title).toBe("New thread");
545 }),
546 ),
547 );
548});