| ... | ... | @@ -1,511 +0,0 @@ |
| 1 | | import { spawn } from "node:child_process"; |
| 2 | | import fs from "node:fs/promises"; |
| 3 | | import path from "node:path"; |
| 4 | | import readline from "node:readline"; |
| 5 | | |
| 6 | | const DEFAULT_PROMPT_TIMEOUT_MS = 120_000; |
| 7 | | const CANCEL_AFTER_MS = 1_500; |
| 8 | | |
| 9 | | function parseArgv(argv) { |
| 10 | | const parsed = { |
| 11 | | outputDir: "", |
| 12 | | workspace: process.cwd(), |
| 13 | | model: "", |
| 14 | | permissionOption: "allow-once", |
| 15 | | }; |
| 16 | | |
| 17 | | for (let i = 0; i < argv.length; i += 1) { |
| 18 | | const token = argv[i]; |
| 19 | | const next = argv[i + 1]; |
| 20 | | if ((token === "--output-dir" || token === "-o") && next) { |
| 21 | | parsed.outputDir = next; |
| 22 | | i += 1; |
| 23 | | continue; |
| 24 | | } |
| 25 | | if ((token === "--workspace" || token === "-w") && next) { |
| 26 | | parsed.workspace = path.resolve(next); |
| 27 | | i += 1; |
| 28 | | continue; |
| 29 | | } |
| 30 | | if ((token === "--model" || token === "-m") && next) { |
| 31 | | parsed.model = next; |
| 32 | | i += 1; |
| 33 | | continue; |
| 34 | | } |
| 35 | | if (token === "--permission-option" && next) { |
| 36 | | parsed.permissionOption = next; |
| 37 | | i += 1; |
| 38 | | } |
| 39 | | } |
| 40 | | |
| 41 | | return parsed; |
| 42 | | } |
| 43 | | |
| 44 | | function nowIso() { |
| 45 | | return new Date().toISOString(); |
| 46 | | } |
| 47 | | |
| 48 | | function safeJsonParse(line) { |
| 49 | | try { |
| 50 | | return { ok: true, value: JSON.parse(line) }; |
| 51 | | } catch (error) { |
| 52 | | return { ok: false, error }; |
| 53 | | } |
| 54 | | } |
| 55 | | |
| 56 | | class AcpProbeClient { |
| 57 | | #child; |
| 58 | | #stdoutRl; |
| 59 | | #stderrRl; |
| 60 | | #nextId = 1; |
| 61 | | #closed = false; |
| 62 | | #pending = new Map(); |
| 63 | | #onMessage; |
| 64 | | #onServerRequest; |
| 65 | | |
| 66 | | constructor({ onMessage, onServerRequest }) { |
| 67 | | this.#onMessage = onMessage; |
| 68 | | this.#onServerRequest = onServerRequest; |
| 69 | | |
| 70 | | this.#child = spawn("agent", ["acp"], { |
| 71 | | stdio: ["pipe", "pipe", "pipe"], |
| 72 | | env: { |
| 73 | | ...process.env, |
| 74 | | NO_COLOR: "1", |
| 75 | | }, |
| 76 | | }); |
| 77 | | |
| 78 | | this.#stdoutRl = readline.createInterface({ input: this.#child.stdout }); |
| 79 | | this.#stderrRl = readline.createInterface({ input: this.#child.stderr }); |
| 80 | | |
| 81 | | this.#stdoutRl.on("line", (line) => this.#handleStdoutLine(line)); |
| 82 | | this.#stderrRl.on("line", (line) => { |
| 83 | | this.#onMessage({ |
| 84 | | ts: nowIso(), |
| 85 | | channel: "stderr", |
| 86 | | line, |
| 87 | | }); |
| 88 | | }); |
| 89 | | |
| 90 | | this.#child.once("exit", (code, signal) => { |
| 91 | | this.#closed = true; |
| 92 | | const reason = `ACP process exited (code=${String(code)}, signal=${String(signal)})`; |
| 93 | | for (const [id, pending] of this.#pending.entries()) { |
| 94 | | this.#pending.delete(id); |
| 95 | | pending.reject(new Error(reason)); |
| 96 | | } |
| 97 | | this.#onMessage({ |
| 98 | | ts: nowIso(), |
| 99 | | channel: "lifecycle", |
| 100 | | event: "exit", |
| 101 | | code, |
| 102 | | signal, |
| 103 | | }); |
| 104 | | }); |
| 105 | | } |
| 106 | | |
| 107 | | async close() { |
| 108 | | if (this.#closed) return; |
| 109 | | try { |
| 110 | | this.#child.stdin.end(); |
| 111 | | } catch { |
| 112 | | // ignored |
| 113 | | } |
| 114 | | this.#child.kill("SIGTERM"); |
| 115 | | await new Promise((resolve) => setTimeout(resolve, 100)); |
| 116 | | if (!this.#closed) { |
| 117 | | this.#child.kill("SIGKILL"); |
| 118 | | } |
| 119 | | } |
| 120 | | |
| 121 | | async send(method, params, { timeoutMs = DEFAULT_PROMPT_TIMEOUT_MS } = {}) { |
| 122 | | if (this.#closed) { |
| 123 | | throw new Error("Cannot send: ACP process is already closed."); |
| 124 | | } |
| 125 | | const id = this.#nextId; |
| 126 | | this.#nextId += 1; |
| 127 | | |
| 128 | | const message = { |
| 129 | | jsonrpc: "2.0", |
| 130 | | id, |
| 131 | | method, |
| 132 | | params, |
| 133 | | }; |
| 134 | | |
| 135 | | this.#onMessage({ |
| 136 | | ts: nowIso(), |
| 137 | | channel: "client->server", |
| 138 | | message, |
| 139 | | }); |
| 140 | | this.#child.stdin.write(`${JSON.stringify(message)}\n`); |
| 141 | | |
| 142 | | return new Promise((resolve, reject) => { |
| 143 | | const timeout = setTimeout(() => { |
| 144 | | this.#pending.delete(id); |
| 145 | | reject(new Error(`Timed out waiting for response to '${method}' (id=${id}).`)); |
| 146 | | }, timeoutMs); |
| 147 | | |
| 148 | | this.#pending.set(id, { |
| 149 | | resolve: (value) => { |
| 150 | | clearTimeout(timeout); |
| 151 | | resolve(value); |
| 152 | | }, |
| 153 | | reject: (error) => { |
| 154 | | clearTimeout(timeout); |
| 155 | | reject(error); |
| 156 | | }, |
| 157 | | }); |
| 158 | | }); |
| 159 | | } |
| 160 | | |
| 161 | | #respond(id, result) { |
| 162 | | if (this.#closed) return; |
| 163 | | const message = { |
| 164 | | jsonrpc: "2.0", |
| 165 | | id, |
| 166 | | result, |
| 167 | | }; |
| 168 | | this.#onMessage({ |
| 169 | | ts: nowIso(), |
| 170 | | channel: "client->server", |
| 171 | | message, |
| 172 | | }); |
| 173 | | this.#child.stdin.write(`${JSON.stringify(message)}\n`); |
| 174 | | } |
| 175 | | |
| 176 | | #handleStdoutLine(line) { |
| 177 | | const parsed = safeJsonParse(line); |
| 178 | | if (!parsed.ok) { |
| 179 | | this.#onMessage({ |
| 180 | | ts: nowIso(), |
| 181 | | channel: "stdout-non-json", |
| 182 | | line, |
| 183 | | }); |
| 184 | | return; |
| 185 | | } |
| 186 | | |
| 187 | | const message = parsed.value; |
| 188 | | this.#onMessage({ |
| 189 | | ts: nowIso(), |
| 190 | | channel: "server->client", |
| 191 | | message, |
| 192 | | }); |
| 193 | | |
| 194 | | if ( |
| 195 | | Object.prototype.hasOwnProperty.call(message, "id") && |
| 196 | | (Object.prototype.hasOwnProperty.call(message, "result") || |
| 197 | | Object.prototype.hasOwnProperty.call(message, "error")) && |
| 198 | | !Object.prototype.hasOwnProperty.call(message, "method") |
| 199 | | ) { |
| 200 | | const pending = this.#pending.get(message.id); |
| 201 | | if (!pending) return; |
| 202 | | this.#pending.delete(message.id); |
| 203 | | if (Object.prototype.hasOwnProperty.call(message, "error")) { |
| 204 | | pending.reject(message.error); |
| 205 | | return; |
| 206 | | } |
| 207 | | pending.resolve(message.result); |
| 208 | | return; |
| 209 | | } |
| 210 | | |
| 211 | | if ( |
| 212 | | Object.prototype.hasOwnProperty.call(message, "id") && |
| 213 | | Object.prototype.hasOwnProperty.call(message, "method") |
| 214 | | ) { |
| 215 | | this.#onServerRequest?.(message, (result) => this.#respond(message.id, result)); |
| 216 | | } |
| 217 | | } |
| 218 | | } |
| 219 | | |
| 220 | | function summarizeTranscript(entries) { |
| 221 | | const summary = { |
| 222 | | counts: { |
| 223 | | notificationsByMethod: {}, |
| 224 | | sessionUpdateByType: {}, |
| 225 | | serverRequestsByMethod: {}, |
| 226 | | permissionDecisions: {}, |
| 227 | | }, |
| 228 | | samples: { |
| 229 | | initializeResult: null, |
| 230 | | authenticateResult: null, |
| 231 | | sessionNewResult: null, |
| 232 | | sessionPromptResultByScenario: {}, |
| 233 | | sessionUpdateByType: {}, |
| 234 | | serverRequestByMethod: {}, |
| 235 | | }, |
| 236 | | stderr: [], |
| 237 | | }; |
| 238 | | |
| 239 | | for (const entry of entries) { |
| 240 | | if (entry.channel === "stderr") { |
| 241 | | summary.stderr.push(entry.line); |
| 242 | | continue; |
| 243 | | } |
| 244 | | if (entry.channel !== "server->client") continue; |
| 245 | | |
| 246 | | const message = entry.message; |
| 247 | | if (!message || typeof message !== "object") continue; |
| 248 | | |
| 249 | | if (typeof message.method === "string" && !Object.prototype.hasOwnProperty.call(message, "id")) { |
| 250 | | summary.counts.notificationsByMethod[message.method] = |
| 251 | | (summary.counts.notificationsByMethod[message.method] ?? 0) + 1; |
| 252 | | if (message.method === "session/update") { |
| 253 | | const updateType = message.params?.update?.sessionUpdate; |
| 254 | | if (typeof updateType === "string") { |
| 255 | | summary.counts.sessionUpdateByType[updateType] = |
| 256 | | (summary.counts.sessionUpdateByType[updateType] ?? 0) + 1; |
| 257 | | if (!summary.samples.sessionUpdateByType[updateType]) { |
| 258 | | summary.samples.sessionUpdateByType[updateType] = message; |
| 259 | | } |
| 260 | | } |
| 261 | | } |
| 262 | | continue; |
| 263 | | } |
| 264 | | |
| 265 | | if (typeof message.method === "string" && Object.prototype.hasOwnProperty.call(message, "id")) { |
| 266 | | summary.counts.serverRequestsByMethod[message.method] = |
| 267 | | (summary.counts.serverRequestsByMethod[message.method] ?? 0) + 1; |
| 268 | | if (!summary.samples.serverRequestByMethod[message.method]) { |
| 269 | | summary.samples.serverRequestByMethod[message.method] = message; |
| 270 | | } |
| 271 | | continue; |
| 272 | | } |
| 273 | | } |
| 274 | | |
| 275 | | for (const entry of entries) { |
| 276 | | if (entry.channel !== "scenario-result") continue; |
| 277 | | if (entry.scenario === "initialize" && !summary.samples.initializeResult) { |
| 278 | | summary.samples.initializeResult = entry.result; |
| 279 | | continue; |
| 280 | | } |
| 281 | | if (entry.scenario === "authenticate" && !summary.samples.authenticateResult) { |
| 282 | | summary.samples.authenticateResult = entry.result; |
| 283 | | continue; |
| 284 | | } |
| 285 | | if (entry.scenario === "session/new" && !summary.samples.sessionNewResult) { |
| 286 | | summary.samples.sessionNewResult = entry.result; |
| 287 | | continue; |
| 288 | | } |
| 289 | | if (entry.scenarioName) { |
| 290 | | summary.samples.sessionPromptResultByScenario[entry.scenarioName] = entry.result; |
| 291 | | } |
| 292 | | } |
| 293 | | |
| 294 | | for (const entry of entries) { |
| 295 | | if (entry.channel !== "permission-decision") continue; |
| 296 | | const optionId = entry.optionId; |
| 297 | | summary.counts.permissionDecisions[optionId] = |
| 298 | | (summary.counts.permissionDecisions[optionId] ?? 0) + 1; |
| 299 | | } |
| 300 | | |
| 301 | | return summary; |
| 302 | | } |
| 303 | | |
| 304 | | async function run() { |
| 305 | | const args = parseArgv(process.argv.slice(2)); |
| 306 | | const allowedPermissionOptions = new Set(["allow-once", "allow-always", "reject-once"]); |
| 307 | | if (!allowedPermissionOptions.has(args.permissionOption)) { |
| 308 | | throw new Error( |
| 309 | | `Invalid --permission-option '${args.permissionOption}'. Expected one of: ${Array.from(allowedPermissionOptions).join(", ")}`, |
| 310 | | ); |
| 311 | | } |
| 312 | | const stamp = nowIso().replaceAll(":", "-"); |
| 313 | | const outputDir = args.outputDir |
| 314 | | ? path.resolve(args.outputDir) |
| 315 | | : path.join(process.cwd(), ".tmp", "acp-probe", stamp); |
| 316 | | await fs.mkdir(outputDir, { recursive: true }); |
| 317 | | |
| 318 | | const transcript = []; |
| 319 | | const pushEntry = (entry) => transcript.push(entry); |
| 320 | | |
| 321 | | let activeScenarioName = null; |
| 322 | | |
| 323 | | const client = new AcpProbeClient({ |
| 324 | | onMessage: (entry) => { |
| 325 | | if ( |
| 326 | | activeScenarioName && |
| 327 | | entry.channel === "server->client" && |
| 328 | | entry.message?.method === "session/update" |
| 329 | | ) { |
| 330 | | transcript.push({ ...entry, scenarioName: activeScenarioName }); |
| 331 | | return; |
| 332 | | } |
| 333 | | transcript.push(entry); |
| 334 | | }, |
| 335 | | onServerRequest: (message, respond) => { |
| 336 | | if (message.method === "session/request_permission") { |
| 337 | | const defaultChoice = args.permissionOption; |
| 338 | | pushEntry({ |
| 339 | | ts: nowIso(), |
| 340 | | channel: "permission-decision", |
| 341 | | requestId: message.id, |
| 342 | | optionId: defaultChoice, |
| 343 | | params: message.params, |
| 344 | | }); |
| 345 | | respond({ |
| 346 | | outcome: { |
| 347 | | outcome: "selected", |
| 348 | | optionId: defaultChoice, |
| 349 | | }, |
| 350 | | }); |
| 351 | | return; |
| 352 | | } |
| 353 | | |
| 354 | | respond({ |
| 355 | | outcome: { |
| 356 | | outcome: "selected", |
| 357 | | optionId: "deny", |
| 358 | | }, |
| 359 | | }); |
| 360 | | pushEntry({ |
| 361 | | ts: nowIso(), |
| 362 | | channel: "server-request-unhandled", |
| 363 | | method: message.method, |
| 364 | | requestId: message.id, |
| 365 | | params: message.params, |
| 366 | | }); |
| 367 | | }, |
| 368 | | }); |
| 369 | | |
| 370 | | try { |
| 371 | | const initializeResult = await client.send("initialize", { |
| 372 | | protocolVersion: 1, |
| 373 | | clientCapabilities: { |
| 374 | | fs: { readTextFile: false, writeTextFile: false }, |
| 375 | | terminal: false, |
| 376 | | }, |
| 377 | | clientInfo: { |
| 378 | | name: "t3-cursor-acp-probe", |
| 379 | | version: "0.1.0", |
| 380 | | }, |
| 381 | | }); |
| 382 | | pushEntry({ |
| 383 | | ts: nowIso(), |
| 384 | | channel: "scenario-result", |
| 385 | | scenario: "initialize", |
| 386 | | result: initializeResult, |
| 387 | | }); |
| 388 | | |
| 389 | | const authenticateResult = await client.send("authenticate", { |
| 390 | | methodId: "cursor_login", |
| 391 | | }); |
| 392 | | pushEntry({ |
| 393 | | ts: nowIso(), |
| 394 | | channel: "scenario-result", |
| 395 | | scenario: "authenticate", |
| 396 | | result: authenticateResult, |
| 397 | | }); |
| 398 | | |
| 399 | | const sessionParams = { |
| 400 | | cwd: args.workspace, |
| 401 | | mcpServers: [], |
| 402 | | ...(args.model ? { model: args.model } : {}), |
| 403 | | }; |
| 404 | | const sessionResult = await client.send("session/new", sessionParams); |
| 405 | | pushEntry({ |
| 406 | | ts: nowIso(), |
| 407 | | channel: "scenario-result", |
| 408 | | scenario: "session/new", |
| 409 | | result: sessionResult, |
| 410 | | }); |
| 411 | | |
| 412 | | const sessionId = sessionResult?.sessionId; |
| 413 | | if (typeof sessionId !== "string" || sessionId.length === 0) { |
| 414 | | throw new Error(`Missing sessionId from session/new response: ${JSON.stringify(sessionResult)}`); |
| 415 | | } |
| 416 | | |
| 417 | | const scenarios = [ |
| 418 | | { |
| 419 | | name: "hello", |
| 420 | | prompt: "Say hello in one sentence.", |
| 421 | | }, |
| 422 | | { |
| 423 | | name: "tooling", |
| 424 | | prompt: |
| 425 | | "Use tools to run `pwd` and then `ls -1 | head -n 8`, and summarize what you found in one paragraph.", |
| 426 | | }, |
| 427 | | { |
| 428 | | name: "cancel", |
| 429 | | prompt: |
| 430 | | "Think for a while and draft a long detailed migration plan with at least 20 bullet points before answering.", |
| 431 | | cancelAfterMs: CANCEL_AFTER_MS, |
| 432 | | }, |
| 433 | | ]; |
| 434 | | |
| 435 | | for (const scenario of scenarios) { |
| 436 | | activeScenarioName = scenario.name; |
| 437 | | const promptParams = { |
| 438 | | sessionId, |
| 439 | | prompt: [{ type: "text", text: scenario.prompt }], |
| 440 | | }; |
| 441 | | |
| 442 | | const promptPromise = client.send("session/prompt", promptParams, { |
| 443 | | timeoutMs: DEFAULT_PROMPT_TIMEOUT_MS, |
| 444 | | }); |
| 445 | | |
| 446 | | if (scenario.cancelAfterMs) { |
| 447 | | setTimeout(() => { |
| 448 | | client |
| 449 | | .send("session/cancel", { sessionId }, { timeoutMs: 15_000 }) |
| 450 | | .then((cancelResult) => { |
| 451 | | pushEntry({ |
| 452 | | ts: nowIso(), |
| 453 | | channel: "scenario-result", |
| 454 | | scenario: "session/cancel", |
| 455 | | scenarioName: scenario.name, |
| 456 | | result: cancelResult, |
| 457 | | }); |
| 458 | | }) |
| 459 | | .catch((error) => { |
| 460 | | pushEntry({ |
| 461 | | ts: nowIso(), |
| 462 | | channel: "scenario-error", |
| 463 | | scenario: "session/cancel", |
| 464 | | scenarioName: scenario.name, |
| 465 | | error: |
| 466 | | error instanceof Error |
| 467 | | ? error.message |
| 468 | | : typeof error === "string" |
| 469 | | ? error |
| 470 | | : JSON.stringify(error), |
| 471 | | }); |
| 472 | | }); |
| 473 | | }, scenario.cancelAfterMs); |
| 474 | | } |
| 475 | | |
| 476 | | const promptResult = await promptPromise; |
| 477 | | pushEntry({ |
| 478 | | ts: nowIso(), |
| 479 | | channel: "scenario-result", |
| 480 | | scenario: "session/prompt", |
| 481 | | scenarioName: scenario.name, |
| 482 | | result: promptResult, |
| 483 | | }); |
| 484 | | activeScenarioName = null; |
| 485 | | await new Promise((resolve) => setTimeout(resolve, 500)); |
| 486 | | } |
| 487 | | } finally { |
| 488 | | await client.close(); |
| 489 | | } |
| 490 | | |
| 491 | | const summary = summarizeTranscript(transcript); |
| 492 | | const transcriptPath = path.join(outputDir, "transcript.ndjson"); |
| 493 | | const summaryPath = path.join(outputDir, "summary.json"); |
| 494 | | |
| 495 | | await fs.writeFile( |
| 496 | | transcriptPath, |
| 497 | | `${transcript.map((entry) => JSON.stringify(entry)).join("\n")}\n`, |
| 498 | | "utf8", |
| 499 | | ); |
| 500 | | await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8"); |
| 501 | | |
| 502 | | process.stdout.write(`ACP probe complete.\n`); |
| 503 | | process.stdout.write(` outputDir: ${outputDir}\n`); |
| 504 | | process.stdout.write(` transcript: ${transcriptPath}\n`); |
| 505 | | process.stdout.write(` summary: ${summaryPath}\n`); |
| 506 | | } |
| 507 | | |
| 508 | | run().catch((error) => { |
| 509 | | process.stderr.write(`ACP probe failed: ${String(error)}\n`); |
| 510 | | process.exitCode = 1; |
| 511 | | }); |