1#!/usr/bin/env node
2// This CLI uses Node argument parsing and random ordering at the application boundary.
3// @effect-diagnostics nodeBuiltinImport:off
4// Run with --model <configured-model> --out /tmp/title-eval.
5// Pass --baseline /tmp/previous-eval/results.json to compare two generation runs.
6// Add --initial to evaluate only the opening request.
7import * as NodeUtil from "node:util";
8import * as NodeCrypto from "node:crypto";
9import * as FetchHttpClient from "effect/http/FetchHttpClient";
10import * as NodeServices from "@effect/platform-node/NodeServices";
11import { CodexSettings, ProviderInstanceId } from "@t3tools/contracts";
12import * as Effect from "effect/Effect";
13import * as Layer from "effect/Layer";
14import * as Duration from "effect/Duration";
15import * as FileSystem from "effect/FileSystem";
16import * as Path from "effect/Path";
17import * as Schema from "effect/Schema";
18import * as CodexTextGeneration from "../src/textGeneration/CodexTextGeneration.ts";
19import { threadTitleEvaluationCases } from "./threadTitleEvaluationCases.ts";
20import {
21 formatThreadTitleContext,
22 type ThreadTitleMessage,
23} from "@t3tools/provider-core/server/threadTitleContext";
24import * as ThreadTitleLinks from "../src/textGeneration/ThreadTitleLinks.ts";
25import * as SourceControlProviderRegistry from "../src/sourceControl/SourceControlProviderRegistry.ts";
26import * as SourceControlBuiltInDrivers from "../src/sourceControl/builtInDrivers.ts";
27import * as ServerSettings from "../src/serverSettings.ts";
28import * as VcsProcess from "../src/vcs/VcsProcess.ts";
29import * as VcsDriverRegistry from "../src/vcs/VcsDriverRegistry.ts";
30import * as VcsProjectConfig from "../src/vcs/VcsProjectConfig.ts";
31import * as GitVcsDriver from "../src/vcs/GitVcsDriver.ts";
32import * as ProcessRunner from "../src/processRunner.ts";
33import * as ServerConfig from "../src/config.ts";
34
35const { values } = NodeUtil.parseArgs({
36 options: {
37 model: { type: "string" },
38 out: { type: "string" },
39 baseline: { type: "string" },
40 initial: { type: "boolean", default: false },
41 },
42});
43if (!values.model || !values.out)
44 throw new Error("Use --model <configured-model> --out <directory>.");
45const model = values.model;
46const outputDirectory = values.out;
47const Results = Schema.fromJsonString(
48 Schema.Array(
49 Schema.Struct({
50 id: Schema.String,
51 title: Schema.String,
52 latencyMs: Schema.Number,
53 linkedContextDigest: Schema.String,
54 }),
55 ),
56);
57const decodeResults = Schema.decodeUnknownEffect(Results);
58const decodeSettings = Schema.decodeUnknownEffect(CodexSettings);
59const encodeReport = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));
60
61await Effect.runPromise(
62 Effect.gen(function* () {
63 const fs = yield* FileSystem.FileSystem;
64 const path = yield* Path.Path;
65 const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-title-evaluation-" });
66 const generation = yield* CodexTextGeneration.makeCodexTextGeneration(
67 yield* decodeSettings({}),
68 );
69 const baseline = values.baseline
70 ? yield* fs.readFileString(values.baseline).pipe(Effect.flatMap(decodeResults))
71 : [];
72 const results = [];
73 const review = [];
74 const answerKey = [];
75 for (const fixture of threadTitleEvaluationCases) {
76 const previous = baseline.find((entry) => entry.id === fixture.id);
77 if (values.baseline && !previous) throw new Error(`Baseline is missing ${fixture.id}.`);
78 const firstMessage: ThreadTitleMessage | undefined = fixture.messages.find(
79 (message) => message.role === "user",
80 );
81 if (!firstMessage) throw new Error(`Fixture ${fixture.id} has no user message.`);
82 const context = formatThreadTitleContext(fixture.messages);
83 const message = values.initial ? firstMessage.text : context.message;
84 const attachments = values.initial ? firstMessage.attachments : context.attachments;
85 const [elapsed, { generated, linkedContextDigest }] = yield* Effect.gen(function* () {
86 const linkedContext = yield* ThreadTitleLinks.resolveThreadTitleLinks({
87 cwd,
88 message,
89 });
90 const linkedContextDigest = NodeCrypto.createHash("sha256")
91 .update(linkedContext ?? "")
92 .digest("hex");
93 if (previous && previous.linkedContextDigest !== linkedContextDigest) {
94 throw new Error(
95 `Linked context changed for ${fixture.id}. Record a new baseline before comparing titles.`,
96 );
97 }
98 const generated = yield* generation.generateThreadTitle({
99 cwd,
100 message,
101 previousTitle: values.initial ? undefined : fixture.previousTitle,
102 attachments,
103 linkedContext,
104 modelSelection: { instanceId: ProviderInstanceId.make("codex"), model },
105 });
106 return { generated, linkedContextDigest };
107 }).pipe(Effect.timed);
108 const oldTitle = previous?.title ?? fixture.previousTitle;
109 const newFirst = NodeCrypto.randomInt(2) === 0;
110 results.push({
111 id: fixture.id,
112 title: generated.title,
113 latencyMs: Duration.toMillis(elapsed),
114 needsRefinement: generated.needsRefinement ?? false,
115 linkedContextDigest,
116 });
117 review.push({
118 id: fixture.id,
119 source: fixture.source,
120 request: fixture.request,
121 rubric: fixture.rubric,
122 A: newFirst ? generated.title : oldTitle,
123 B: newFirst ? oldTitle : generated.title,
124 preferred: "",
125 subjectAccuracy: "",
126 recognitionAmongNearbyThreads: "",
127 });
128 answerKey.push({ id: fixture.id, candidate: newFirst ? "A" : "B" });
129 }
130 yield* fs.makeDirectory(outputDirectory, { recursive: true });
131 for (const [name, report] of [
132 ["results", results],
133 ["review", review],
134 ["answer-key", answerKey],
135 ] as const) {
136 yield* fs.writeFileString(
137 path.join(outputDirectory, `${name}.json`),
138 yield* encodeReport(report),
139 );
140 }
141 yield* Effect.log(
142 `Wrote ${results.length} cases to ${outputDirectory}. Score review.json before opening answer-key.json. Latency is in results.json.`,
143 );
144 }).pipe(
145 Effect.provide(
146 Layer.mergeAll(
147 ProcessRunner.layer,
148 SourceControlProviderRegistry.layer.pipe(
149 Layer.provide(SourceControlBuiltInDrivers.layer),
150 // Default settings: no saved Bitbucket token, gh's own GitHub account choice.
151 Layer.provide(ServerSettings.layerTest()),
152 Layer.provide(VcsDriverRegistry.layer.pipe(Layer.provide(VcsProjectConfig.layer))),
153 Layer.provide(GitVcsDriver.layer),
154 Layer.provide(VcsProcess.layer),
155 Layer.provide(FetchHttpClient.layer),
156 ),
157 ).pipe(
158 Layer.provideMerge(
159 ServerConfig.layerTest(process.cwd(), { prefix: "t3-title-evaluation-state-" }),
160 ),
161 Layer.provideMerge(NodeServices.layer),
162 ),
163 ),
164 Effect.scoped,
165 ),
166);