1import type { ThreadTitleMessage } from "@t3tools/provider-core/server/threadTitleContext";
2
3// Public PR subjects and existing title scenarios. Repeated text adds context pressure.
4export const threadTitleEvaluationCases = [
5 {
6 id: "linked-reset-credits",
7 source: "https://github.com/pingdotgg/t3code/pull/10462",
8 request: "Review the reset credit routing change.",
9 previousTitle: "Review PR 10462",
10 messages: [{ role: "user", text: "Review https://github.com/pingdotgg/t3code/pull/10462" }],
11 rubric: "Name reset credit routing. Distinguish it from displaying credit balances.",
12 },
13 {
14 id: "onboarding-merge",
15 source: "https://github.com/pingdotgg/t3code/pull/10465",
16 request: "Make onboarding one shared wizard across computers, then merge when green.",
17 previousTitle: "Finish onboarding PR",
18 messages: [
19 { role: "user", text: "Make onboarding one shared wizard across computers." },
20 {
21 role: "assistant",
22 text: "The wizard now handles pairing, agent selection, and project import.",
23 },
24 { role: "user", text: "File a PR and merge it when green." },
25 ],
26 rubric: "Keep the multi-computer onboarding subject. Do not title it after merging.",
27 },
28 {
29 id: "vague-opening",
30 source: "Existing lazy thread feed title scenario",
31 request: "A failing test is later identified as a lazy thread feed mismatch.",
32 previousTitle: "Fix failing test",
33 messages: [
34 { role: "user", text: "Fix this failing test." },
35 {
36 role: "assistant",
37 text: "The lazy thread feed test expects a full message body before the client requests it.",
38 },
39 ],
40 rubric: "Name the lazy thread feed test. Do not invent a wider mobile regression.",
41 },
42 {
43 id: "scope-change",
44 source: "Title context budget scenario",
45 request: "Change the goal from QR layout to pairing expiry, despite long assistant replies.",
46 previousTitle: "Improve QR layout",
47 messages: [
48 { role: "user", text: "Improve QR sharing layout." },
49 {
50 role: "user",
51 text: "Change of plan. Fix pairing token expiry. Keep remote access working.",
52 },
53 {
54 role: "assistant",
55 text: "The token expires before redemption. " + "Implementation detail. ".repeat(800),
56 },
57 { role: "user", text: "Ship it." },
58 ],
59 rubric: "Name pairing expiry and honor the explicit scope change.",
60 },
61 {
62 id: "review-umbrella",
63 source: "Existing subagent monitoring title scenario",
64 request: "Review subagent monitoring risks. A Codex roster issue is one finding.",
65 previousTitle: "Review subagent monitoring risks",
66 messages: [
67 { role: "user", text: "Review subagent monitoring risks." },
68 {
69 role: "assistant",
70 text: "One finding is a stale Codex roster. " + "Roster detail. ".repeat(800),
71 },
72 { role: "user", text: "Fix the findings and babysit CI." },
73 ],
74 rubric: "Preserve the monitoring review scope. The previous title can stay unchanged.",
75 },
76 {
77 id: "long-opening",
78 source: "Title message truncation scenario",
79 request: "Investigate Android pairing while preserving the iOS flow.",
80 previousTitle: "Inspect logs",
81 messages: [
82 {
83 role: "user",
84 text:
85 "Investigate Android pairing. " +
86 "Connection logs. ".repeat(800) +
87 " Preserve the iOS pairing flow.",
88 },
89 ],
90 rubric: "Name Android pairing. Logs are supporting evidence.",
91 },
92 {
93 id: "research",
94 source: "Maintainer title generation request",
95 request: "How can we improve title generation in T3 Code?",
96 previousTitle: "Research title gen improvements",
97 messages: [
98 { role: "user", text: "How can we improve title gen further in T3 Code?" },
99 {
100 role: "assistant",
101 text: "Prioritize user messages, refine vague titles once, and resolve PR subjects.",
102 },
103 {
104 role: "user",
105 text: "Make these changes and file a PR. Babysit until everything is green.",
106 },
107 ],
108 rubric: "Keep title generation as the subject. Do not focus on filing the PR.",
109 },
110] satisfies ReadonlyArray<{
111 id: string;
112 source: string;
113 request: string;
114 previousTitle: string;
115 messages: ReadonlyArray<ThreadTitleMessage>;
116 rubric: string;
117}>;