1import { directiveFromMarkdown } from "mdast-util-directive";
2import { directive } from "micromark-extension-directive";
3import {
4 markdownLineEnding,
5 unicodePunctuation,
6 unicodeWhitespace,
7} from "micromark-util-character";
8import type { Construct, Extension, Tokenizer } from "micromark-util-types";
9import remarkParse from "remark-parse";
10import { unified, type Processor } from "unified";
11
12import {
13 codexArtifactTemplatePresentationLabel,
14 resolveCodexArtifactTemplate,
15 type CodexArtifactTemplate,
16} from "./codexArtifactTemplates.ts";
17import { codexFileCitationMarkdown, resolveCodexFileCitationLink } from "./codexFileCitations.ts";
18
19const COLON = 58;
20const DASH = 45;
21const UNDERSCORE = 95;
22const CODEX_FILE_CITATION_NAME = "codex-file-citation";
23const CODEX_FOLLOW_UP_NAME = "codex-followup";
24const CODEX_ARTIFACT_TEMPLATE_NAME = "artifact-template";
25
26export const CODEX_ARTIFACT_TEMPLATE_HAST_PROPERTIES = [
27 "dataCodexArtifactTemplate",
28 "dataArtifactKind",
29 "dataDisplayName",
30 "dataGalleryKind",
31 "dataSkillDirectory",
32 "dataSkillName",
33] as const;
34
35interface MarkdownPosition {
36 readonly start: { readonly offset?: number };
37 readonly end: { readonly offset?: number };
38}
39
40interface MarkdownAstNode {
41 type?: string;
42 name?: string;
43 value?: string;
44 url?: string;
45 attributes?: Readonly<Record<string, string | null>>;
46 position?: MarkdownPosition;
47 data?: {
48 codexArtifactTemplate?: CodexArtifactTemplate;
49 codexFileCitationMarkdown?: string;
50 codexFollowUp?: true;
51 hName?: string;
52 hProperties?: Record<string, unknown>;
53 };
54 children?: MarkdownAstNode[];
55}
56
57interface MarkdownFile {
58 readonly value: unknown;
59}
60
61export type CodexArtifactTemplateMarkdownSegment =
62 | { readonly kind: "markdown"; readonly markdown: string; readonly sourceOffset: number }
63 | {
64 readonly kind: "artifact-template";
65 readonly sourceOffset: number;
66 readonly template: CodexArtifactTemplate;
67 };
68
69function asConstruct(value: Construct | Construct[] | undefined, label: string): Construct {
70 const construct = Array.isArray(value) ? value[0] : value;
71 if (!construct) throw new Error(`Missing ${label} directive construct`);
72 return construct;
73}
74
75function directiveNameEnds(code: number | null): boolean {
76 return (
77 code === null ||
78 markdownLineEnding(code) ||
79 unicodeWhitespace(code) ||
80 (unicodePunctuation(code) && code !== DASH && code !== UNDERSCORE)
81 );
82}
83
84function directiveNameGate(markerCount: number, name: string): Construct {
85 const tokenize: Tokenizer = (effects, ok, nok) => {
86 let markerIndex = 0;
87 let nameIndex = 0;
88
89 return marker;
90
91 function marker(code: number | null) {
92 if (code !== COLON) return nok(code);
93 if (markerIndex === 0) effects.enter("data");
94 effects.consume(code);
95 markerIndex += 1;
96 return markerIndex === markerCount ? nameCharacter : marker;
97 }
98
99 function nameCharacter(code: number | null) {
100 if (code !== name.charCodeAt(nameIndex)) return nok(code);
101 effects.consume(code);
102 nameIndex += 1;
103 return nameIndex === name.length ? afterName : nameCharacter;
104 }
105
106 function afterName(code: number | null) {
107 effects.exit("data");
108 return directiveNameEnds(code) ? ok(code) : nok(code);
109 }
110 };
111
112 return { partial: true, tokenize };
113}
114
115function restrictedDirective(construct: Construct, markerCount: number, name: string): Construct {
116 const gate = directiveNameGate(markerCount, name);
117 return {
118 ...construct,
119 tokenize(effects, ok, nok) {
120 return effects.check(gate, construct.tokenize.call(this, effects, ok, nok), nok);
121 },
122 };
123}
124
125function codexDirectiveSyntax(): Extension {
126 const genericSyntax = directive();
127 const textDirective = asConstruct(genericSyntax.text?.[COLON], CODEX_FILE_CITATION_NAME);
128 const flowDirectives = genericSyntax.flow?.[COLON];
129 const leafDirective = Array.isArray(flowDirectives)
130 ? flowDirectives.find((construct) => construct.concrete !== true)
131 : flowDirectives;
132 if (!leafDirective)
133 throw new Error(`Missing ${CODEX_ARTIFACT_TEMPLATE_NAME} directive construct`);
134
135 return {
136 text: {
137 [COLON]: [
138 restrictedDirective(textDirective, 1, CODEX_FILE_CITATION_NAME),
139 restrictedDirective(textDirective, 1, CODEX_FOLLOW_UP_NAME),
140 ],
141 },
142 flow: {
143 [COLON]: restrictedDirective(leafDirective, 2, CODEX_ARTIFACT_TEMPLATE_NAME),
144 },
145 };
146}
147
148const CODEX_DIRECTIVE_SYNTAX = codexDirectiveSyntax();
149const CODEX_DIRECTIVE_FROM_MARKDOWN = directiveFromMarkdown();
150
151function sourceForNode(node: MarkdownAstNode, source: string): string {
152 const start = node.position?.start.offset;
153 const end = node.position?.end.offset;
154 return start === undefined || end === undefined ? "" : source.slice(start, end);
155}
156
157function textContent(node: MarkdownAstNode): string {
158 return node.value ?? (node.children ?? []).map(textContent).join("");
159}
160
161function sourceForDirective(node: MarkdownAstNode, source: string, marker: ":" | "::"): string {
162 const prefix = `${marker}${node.name ?? ""}`;
163 const slicedSource = sourceForNode(node, source);
164 if (slicedSource.startsWith(prefix)) return slicedSource;
165
166 const label = node.children?.length ? `[${textContent(node)}]` : "";
167 const attributes = Object.entries(node.attributes ?? {}).map(([name, value]) =>
168 value === null ? name : `${name}=${JSON.stringify(value)}`,
169 );
170 return `${prefix}${label}${attributes.length === 0 ? "" : `{${attributes.join(" ")}}`}`;
171}
172
173function restoreTextDirective(node: MarkdownAstNode, source: string): void {
174 node.type = "text";
175 node.value = sourceForDirective(node, source, ":");
176 delete node.name;
177 delete node.attributes;
178 delete node.url;
179 delete node.data;
180 delete node.children;
181}
182
183function restoreLeafDirective(node: MarkdownAstNode, source: string): void {
184 const value = sourceForDirective(node, source, "::");
185 node.type = "paragraph";
186 node.children = [
187 { type: "text", value, ...(node.position === undefined ? {} : { position: node.position }) },
188 ];
189 delete node.name;
190 delete node.attributes;
191 delete node.value;
192 delete node.url;
193 delete node.data;
194}
195
196function renderFileCitation(node: MarkdownAstNode, source: string, insideLink: boolean): void {
197 const citation = resolveCodexFileCitationLink(node.attributes);
198 if (!citation || insideLink) {
199 restoreTextDirective(node, source);
200 return;
201 }
202
203 node.type = "link";
204 node.url = citation.href;
205 node.children = [{ type: "text", value: citation.label }];
206 node.data = { codexFileCitationMarkdown: codexFileCitationMarkdown(citation) };
207 delete node.name;
208 delete node.attributes;
209 delete node.value;
210}
211
212/** Shows only the label. T3 has no suggestion UI, so the hidden prompt is not shown. */
213function renderFollowUp(node: MarkdownAstNode, source: string): void {
214 if (!node.attributes?.prompt?.trim() || !textContent(node).trim()) {
215 restoreTextDirective(node, source);
216 return;
217 }
218
219 node.data = { codexFollowUp: true, hName: "span" };
220 delete node.name;
221 delete node.attributes;
222}
223
224function renderArtifactTemplate(node: MarkdownAstNode, source: string): void {
225 const template = resolveCodexArtifactTemplate(node.attributes);
226 if (!template) {
227 restoreLeafDirective(node, source);
228 return;
229 }
230
231 node.type = "paragraph";
232 node.children = [];
233 node.data = {
234 codexArtifactTemplate: template,
235 hName: "div",
236 hProperties: {
237 dataCodexArtifactTemplate: "true",
238 dataArtifactKind: template.artifactKind,
239 dataDisplayName: template.displayName,
240 ...(template.galleryKind === undefined ? {} : { dataGalleryKind: template.galleryKind }),
241 dataSkillDirectory: template.skillDirectory,
242 dataSkillName: template.skillName,
243 },
244 };
245 delete node.name;
246 delete node.attributes;
247 delete node.value;
248 delete node.url;
249}
250
251function transformCodexDirectives(node: MarkdownAstNode, source: string, insideLink = false): void {
252 if (node.type === "textDirective" && node.name === CODEX_FILE_CITATION_NAME) {
253 renderFileCitation(node, source, insideLink);
254 return;
255 }
256 if (node.type === "textDirective" && node.name === CODEX_FOLLOW_UP_NAME) {
257 renderFollowUp(node, source);
258 }
259 if (node.type === "leafDirective" && node.name === CODEX_ARTIFACT_TEMPLATE_NAME) {
260 renderArtifactTemplate(node, source);
261 return;
262 }
263
264 const childrenInsideLink = insideLink || node.type === "link" || node.type === "linkReference";
265 for (const child of node.children ?? []) {
266 transformCodexDirectives(child, source, childrenInsideLink);
267 }
268}
269
270/** Adds grammar only for the directives emitted by Codex, then renders them as mdast. */
271function attachCodexDirectives(this: Processor) {
272 const data = this.data();
273 const micromarkExtensions = data.micromarkExtensions ?? (data.micromarkExtensions = []);
274 const fromMarkdownExtensions = data.fromMarkdownExtensions ?? (data.fromMarkdownExtensions = []);
275 micromarkExtensions.push(CODEX_DIRECTIVE_SYNTAX);
276 fromMarkdownExtensions.push(CODEX_DIRECTIVE_FROM_MARKDOWN);
277
278 return (tree: unknown, file: MarkdownFile) => {
279 transformCodexDirectives(tree as MarkdownAstNode, String(file.value));
280 };
281}
282
283export const remarkCodexDirectives = attachCodexDirectives;
284
285const directiveParser = unified().use(remarkParse).use(remarkCodexDirectives).freeze();
286
287function parseCodexMarkdown(markdown: string): MarkdownAstNode {
288 return directiveParser.runSync(directiveParser.parse(markdown), {
289 value: markdown,
290 }) as MarkdownAstNode;
291}
292
293interface DirectiveMatch {
294 readonly start: number;
295 readonly end: number;
296 readonly markdown?: string;
297 readonly template?: CodexArtifactTemplate;
298 readonly labelRange?: readonly [start: number, end: number];
299}
300
301function collectDirectiveMatches(node: MarkdownAstNode, matches: DirectiveMatch[]): void {
302 const start = node.position?.start.offset;
303 const end = node.position?.end.offset;
304 if (start !== undefined && end !== undefined) {
305 if (node.data?.codexFileCitationMarkdown !== undefined) {
306 matches.push({ start, end, markdown: node.data.codexFileCitationMarkdown });
307 return;
308 }
309 if (node.data?.codexArtifactTemplate !== undefined) {
310 matches.push({ start, end, template: node.data.codexArtifactTemplate });
311 return;
312 }
313 const labelStart = node.children?.[0]?.position?.start.offset;
314 const labelEnd = node.children?.at(-1)?.position?.end.offset;
315 if (node.data?.codexFollowUp && labelStart !== undefined && labelEnd !== undefined) {
316 matches.push({ start, end, labelRange: [labelStart, labelEnd] });
317 return;
318 }
319 }
320 for (const child of node.children ?? []) collectDirectiveMatches(child, matches);
321}
322
323function renderDirectiveMatches(
324 markdown: string,
325 replacementFor: (match: DirectiveMatch) => string | undefined,
326): string {
327 const matches: DirectiveMatch[] = [];
328 collectDirectiveMatches(parseCodexMarkdown(markdown), matches);
329 let rendered = markdown;
330 for (const match of matches.sort((left, right) => right.start - left.start)) {
331 const replacement = match.labelRange
332 ? markdown.slice(...match.labelRange)
333 : replacementFor(match);
334 if (replacement !== undefined) {
335 rendered = rendered.slice(0, match.start) + replacement + rendered.slice(match.end);
336 }
337 }
338 return rendered;
339}
340
341/** Native Markdown renderers use this adapter because they cannot consume a Remark tree. */
342export function renderCodexInlineDirectivesAsMarkdown(markdown: string): string {
343 if (
344 !markdown.includes(`:${CODEX_FILE_CITATION_NAME}`) &&
345 !markdown.includes(`:${CODEX_FOLLOW_UP_NAME}`)
346 ) {
347 return markdown;
348 }
349
350 return renderDirectiveMatches(markdown, (match) => match.markdown);
351}
352
353/** Matches the Markdown emitted when users copy rendered Codex directive UI. */
354export function renderCodexDirectivesForCopy(markdown: string): string {
355 if (
356 !markdown.includes(`:${CODEX_FILE_CITATION_NAME}`) &&
357 !markdown.includes(`:${CODEX_FOLLOW_UP_NAME}`) &&
358 !markdown.includes(`::${CODEX_ARTIFACT_TEMPLATE_NAME}`)
359 ) {
360 return markdown;
361 }
362
363 return renderDirectiveMatches(markdown, (match) => {
364 if (match.markdown !== undefined) return match.markdown;
365 if (match.template === undefined) return undefined;
366 return `${match.template.displayName} (${codexArtifactTemplatePresentationLabel(match.template.artifactKind)})`;
367 });
368}
369
370/** Native renderers split cards out because they cannot host a view inside Markdown text. */
371export function splitCodexArtifactTemplateMarkdown(
372 markdown: string,
373): ReadonlyArray<CodexArtifactTemplateMarkdownSegment> {
374 if (!markdown.includes(`::${CODEX_ARTIFACT_TEMPLATE_NAME}`)) {
375 return [{ kind: "markdown", markdown, sourceOffset: 0 }];
376 }
377
378 const matches: DirectiveMatch[] = [];
379 collectDirectiveMatches(parseCodexMarkdown(markdown), matches);
380 const templates = matches
381 .filter(
382 (match): match is DirectiveMatch & { readonly template: CodexArtifactTemplate } =>
383 match.template !== undefined,
384 )
385 .sort((left, right) => left.start - right.start);
386 if (templates.length === 0) {
387 return [{ kind: "markdown", markdown, sourceOffset: 0 }];
388 }
389
390 const segments: CodexArtifactTemplateMarkdownSegment[] = [];
391 let cursor = 0;
392 for (const match of templates) {
393 if (match.start > cursor) {
394 segments.push({
395 kind: "markdown",
396 markdown: markdown.slice(cursor, match.start),
397 sourceOffset: cursor,
398 });
399 }
400 segments.push({
401 kind: "artifact-template",
402 sourceOffset: match.start,
403 template: match.template,
404 });
405 cursor = match.end;
406 }
407 if (cursor < markdown.length) {
408 segments.push({ kind: "markdown", markdown: markdown.slice(cursor), sourceOffset: cursor });
409 }
410 return segments;
411}
412
413export function artifactTemplateFromHastProperties(
414 properties: Readonly<Record<string, unknown>> | null | undefined,
415): CodexArtifactTemplate | null {
416 if (properties?.dataCodexArtifactTemplate !== "true") return null;
417 const stringProperty = (name: string) => {
418 const value = properties[name];
419 return typeof value === "string" ? value : undefined;
420 };
421 return resolveCodexArtifactTemplate({
422 artifact_kind: stringProperty("dataArtifactKind"),
423 display_name: stringProperty("dataDisplayName"),
424 gallery_kind: stringProperty("dataGalleryKind"),
425 skill_directory: stringProperty("dataSkillDirectory"),
426 skill_name: stringProperty("dataSkillName"),
427 });
428}