1import { SOCIAL_REDIRECTS } from "./social-redirects.ts";
2import type { WorkerEnv } from "../alchemy.run.ts";
3import { referenceDestination } from "./reference-links.ts";
4
5// Minimal `HTMLRewriter` shape — the workers runtime exposes it as a
6// global, but we don't pull in `@cloudflare/workers-types`, so declare
7// just what this file uses.
8declare class HTMLRewriter {
9 on(
10 selector: string,
11 handler: { element(el: HTMLRewriterElement): void },
12 ): HTMLRewriter;
13 transform(response: Response): Response;
14}
15interface HTMLRewriterElement {
16 getAttribute(name: string): string | null;
17 setAttribute(name: string, value: string): HTMLRewriterElement;
18}
19
20/**
21 * `https://alchemy.run` is the one indexable origin. Every other host this
22 * worker answers on — `main.alchemy.run`, PR previews on `*.workers.dev`,
23 * future aliases — is a mirror of the same build:
24 *
25 * - `<link rel="canonical">` is baked at build time against `site` and is left
26 * alone, so mirrors point search engines back at production.
27 * - Every mirror response carries `X-Robots-Tag: noindex` and its `robots.txt`
28 * advertises no sitemap, so a mirror URL that leaks (a PR comment, a shared
29 * link) never becomes a duplicate search result.
30 * - `og:*` / `twitter:*` card URLs ARE rewritten to the request host, so a
31 * Slack/X unfurl of a preview URL shows that preview's card, not
32 * production's.
33 */
34const CANONICAL_HOST = "alchemy.run";
35const CANONICAL_ORIGIN = `https://${CANONICAL_HOST}`;
36
37/**
38 * The v1 docs moved to their own host. v1-era URLs (`/docs/...`, and the
39 * pre-hub `/guides/...` / `/providers/...` / `/concepts/...` trees) are still
40 * in search indexes; instead of 404ing them we probe v1 and 301 when it has
41 * the page.
42 */
43const V1_ORIGIN = "https://v1.alchemy.run";
44const V1_PREFIXES = ["/docs/", "/guides/", "/providers/", "/concepts/"];
45
46/**
47 * 301s for the docs restructure (guides/tutorials moved into per-cloud hubs).
48 * Keys and targets are extensionless; `.md` requests (agents fetching raw
49 * markdown) are redirected to the target's `.md` form with any fragment
50 * dropped.
51 */
52const REDIRECTS: Record<string, string> = {
53 "/tutorial/part-1": "/cloudflare/tutorial/part-1",
54 "/tutorial/part-2": "/cloudflare/tutorial/part-2",
55 "/tutorial/part-3": "/cloudflare/tutorial/part-3",
56 "/tutorial/part-4": "/cloudflare/tutorial/part-4",
57 "/tutorial/part-5": "/cloudflare/tutorial/part-5",
58 "/tutorial/cloudflare/compute/durable-objects":
59 "/cloudflare/compute/durable-objects",
60 "/tutorial/cloudflare/data/hyperdrive": "/cloudflare/data/hyperdrive",
61 "/tutorial/cloudflare/queue-consumer": "/cloudflare/messaging/queues",
62 "/tutorial/cloudflare/rpc-durable-object":
63 "/cloudflare/compute/durable-objects#schemaless-rpc",
64 "/tutorial/cloudflare/rpc-worker":
65 "/cloudflare/compute/workers#call-another-worker",
66 "/tutorial/cloudflare/ai-gateway": "/cloudflare/ai/ai-gateway",
67 "/tutorial/cloudflare/ai-search": "/cloudflare/ai/ai-search",
68 "/tutorial/cloudflare/artifacts": "/cloudflare/data/artifacts",
69 "/tutorial/cloudflare/branch-from-shared-database":
70 "/cloudflare/data/branch-from-shared-database",
71 "/tutorial/cloudflare/compute/containers":
72 "/cloudflare/compute/run-a-container",
73 "/tutorial/cloudflare/cross-worker-durable-object":
74 "/cloudflare/compute/cross-worker-durable-object",
75 "/tutorial/cloudflare/drizzle": "/cloudflare/data/drizzle",
76 "/tutorial/cloudflare/hibernatable-websockets":
77 "/cloudflare/compute/hibernatable-websockets",
78 "/tutorial/cloudflare/vite-spa": "/cloudflare/frontend/vite-spa",
79 "/tutorial/cloudflare/compute/workflows":
80 "/cloudflare/compute/add-a-workflow",
81 "/tutorial/aws/compute/lambda": "/aws/compute/lambda",
82 "/tutorial/aws/data/dynamodb": "/aws/data/dynamodb",
83 "/tutorial/aws/messaging/sqs": "/aws/messaging/sqs",
84 "/tutorial/aws/data/s3": "/aws/data/s3",
85 "/tutorial/aws/messaging/kinesis": "/aws/messaging/kinesis",
86 "/tutorial/aws/api-gateway": "/aws/apis/api-gateway",
87 "/tutorial/aws/dynamodb-streams": "/aws/messaging/dynamodb-streams",
88 "/tutorial/aws/s3-events": "/aws/messaging/s3-events",
89 "/guides/effect-http-api": "/cloudflare/apis/effect-http-api",
90 "/guides/effect-rpc": "/cloudflare/apis/effect-rpc",
91 "/guides/effect-ai": "/cloudflare/ai/effect-ai",
92 "/guides/frontends": "/cloudflare/frontend/frontends",
93 "/guides/shared-database": "/cloudflare/data/shared-database",
94 // Absorbed into the Security & secrets block pages.
95 "/guides/secrets": "/cloudflare/security/secrets-env",
96 "/guides/stack-references": "/infrastructure-as-code/references",
97 // The CLI reference became its own hub tab.
98 "/guides/cli": "/cli",
99 "/aws/guides/secrets": "/aws/security/secrets-env",
100 // The Integrations tab was replaced by per-provider hubs (PlanetScale and
101 // Neon promoted to top-level tabs; Axiom and GitHub in the More menu).
102 // Section-folder alignment: file locations now mirror the sidebar taxonomy.
103 "/aws/choosing-a-runtime": "/aws/compute/choosing-a-runtime",
104 "/aws/cloudwatch": "/aws/observability/cloudwatch",
105 "/aws/dynamodb": "/aws/data/dynamodb",
106 "/aws/ec2": "/aws/compute/ec2",
107 "/aws/ecs": "/aws/compute/ecs",
108 "/aws/eks": "/aws/compute/eks",
109 "/aws/eventbridge": "/aws/messaging/eventbridge",
110 "/aws/guides/api-gateway": "/aws/apis/api-gateway",
111 "/aws/guides/custom-domains": "/aws/networking/custom-domains",
112 "/aws/guides/dynamodb-streams": "/aws/messaging/dynamodb-streams",
113 "/aws/guides/effect-http-api": "/aws/apis/effect-http-api",
114 "/aws/guides/effect-rpc": "/aws/apis/effect-rpc",
115 "/aws/guides/microvms": "/aws/compute/microvms",
116 "/aws/guides/s3-events": "/aws/messaging/s3-events",
117 "/aws/guides/static-site": "/aws/frontend/static-site",
118 "/aws/kinesis": "/aws/messaging/kinesis",
119 "/aws/lambda": "/aws/compute/lambda",
120 "/aws/rds": "/aws/data/rds",
121 "/aws/s3": "/aws/data/s3",
122 "/aws/secrets-env": "/aws/security/secrets-env",
123 "/aws/sns": "/aws/messaging/sns",
124 "/aws/sqs": "/aws/messaging/sqs",
125 "/aws/websites": "/aws/frontend/websites",
126 "/axiom/ingest": "/axiom/data/ingest",
127 "/cloudflare/containers": "/cloudflare/compute/containers",
128 "/cloudflare/d1": "/cloudflare/data/d1",
129 "/cloudflare/domains": "/cloudflare/networking/domains",
130 "/cloudflare/durable-objects": "/cloudflare/compute/durable-objects",
131 "/cloudflare/guides/ai-gateway": "/cloudflare/ai/ai-gateway",
132 "/cloudflare/guides/ai-search": "/cloudflare/ai/ai-search",
133 "/cloudflare/guides/analytics-engine":
134 "/cloudflare/observability/analytics-engine",
135 "/cloudflare/guides/artifacts": "/cloudflare/data/artifacts",
136 "/cloudflare/guides/axiom-observability":
137 "/cloudflare/observability/axiom-observability",
138 "/cloudflare/guides/branch-from-shared-database":
139 "/cloudflare/data/branch-from-shared-database",
140 "/cloudflare/guides/browser-rendering":
141 "/cloudflare/compute/browser-rendering",
142 "/cloudflare/guides/containers": "/cloudflare/compute/run-a-container",
143 "/cloudflare/guides/cron": "/cloudflare/messaging/cron",
144 "/cloudflare/guides/cross-worker-durable-object":
145 "/cloudflare/compute/cross-worker-durable-object",
146 "/cloudflare/guides/custom-domains": "/cloudflare/networking/custom-domains",
147 "/cloudflare/guides/drizzle": "/cloudflare/data/drizzle",
148 "/cloudflare/guides/effect-ai": "/cloudflare/ai/effect-ai",
149 "/cloudflare/guides/effect-http-api": "/cloudflare/apis/effect-http-api",
150 "/cloudflare/guides/effect-rpc": "/cloudflare/apis/effect-rpc",
151 "/cloudflare/guides/email": "/cloudflare/email/send-and-receive",
152 "/cloudflare/guides/frontends": "/cloudflare/frontend/frontends",
153 "/cloudflare/guides/github-events": "/cloudflare/messaging/github-events",
154 "/cloudflare/guides/hibernatable-websockets":
155 "/cloudflare/compute/hibernatable-websockets",
156 "/cloudflare/guides/release-agent": "/cloudflare/ai/release-agent",
157 "/cloudflare/guides/secrets-store": "/cloudflare/security/secrets-store",
158 "/cloudflare/guides/shared-database": "/cloudflare/data/shared-database",
159 "/cloudflare/guides/tunnel": "/cloudflare/networking/tunnel",
160 "/cloudflare/guides/turnstile": "/cloudflare/security/turnstile",
161 "/cloudflare/guides/vectorize": "/cloudflare/ai/vectorize",
162 "/cloudflare/guides/vite-spa": "/cloudflare/frontend/vite-spa",
163 "/cloudflare/guides/workers-for-platforms":
164 "/cloudflare/compute/workers-for-platforms",
165 "/cloudflare/guides/workflows": "/cloudflare/compute/add-a-workflow",
166 "/cloudflare/hyperdrive": "/cloudflare/data/hyperdrive",
167 "/cloudflare/kv": "/cloudflare/data/kv",
168 "/cloudflare/queues": "/cloudflare/messaging/queues",
169 "/cloudflare/r2": "/cloudflare/data/r2",
170 "/cloudflare/secrets-env": "/cloudflare/security/secrets-env",
171 "/cloudflare/workers": "/cloudflare/compute/workers",
172 "/cloudflare/workflows": "/cloudflare/compute/workflows",
173 "/concepts/action": "/infrastructure-as-code/action",
174 "/concepts/binding": "/infrastructure-as-effects/binding",
175 "/concepts/layers": "/infrastructure-as-effects/layers",
176 "/concepts/local-development": "/environments/local-development",
177 "/concepts/observability": "/testing/observability",
178 "/concepts/outputs": "/infrastructure-as-code/outputs",
179 "/concepts/phases": "/infrastructure-as-effects/phases",
180 "/concepts/platform": "/infrastructure-as-effects/runtime",
181 "/infrastructure-as-effects/platform": "/infrastructure-as-effects/runtime",
182 "/infrastructure-as-effects/functions-and-servers":
183 "/infrastructure-as-effects/runtime",
184 "/rpc": "/apis",
185 "/rpc/schemaless": "/apis/schemaless",
186 "/rpc/effect-rpc": "/apis/effect-rpc",
187 "/rpc/effect-http": "/apis/effect-http",
188 "/concepts/profiles": "/environments/profiles",
189 "/concepts/provider": "/infrastructure-as-code/provider",
190 "/concepts/references": "/infrastructure-as-code/references",
191 "/concepts/resource": "/infrastructure-as-code/resource",
192 "/concepts/resource-lifecycle": "/infrastructure-as-code/resource-lifecycle",
193 "/concepts/secrets": "/environments/secrets",
194 "/concepts/stack": "/infrastructure-as-code/stack",
195 "/concepts/stages": "/environments/stages",
196 "/concepts/state-store": "/state-store",
197 "/concepts/test-harness": "/testing/test-harness",
198 "/concepts/testing": "/testing",
199 "/fly/frontend": "/fly/frontend/websites",
200 "/fly/websites": "/fly/frontend/websites",
201 "/guides/ci": "/environments/ci",
202 "/guides/circular-bindings": "/infrastructure-as-effects/circular-bindings",
203 "/guides/custom-provider": "/infrastructure-as-code/custom-provider",
204 "/guides/custom-state-store": "/state-store/custom-state-store",
205 "/guides/file-layout": "/project-structure/file-layout",
206 "/guides/infrastructure-layers": "/infrastructure-as-effects/layers",
207 "/infrastructure-as-effects/infrastructure-layers":
208 "/infrastructure-as-effects/layers",
209 "/guides/migrating-from-v1": "/migrating-from-v1",
210 "/guides/monorepo": "/project-structure/monorepo",
211 "/guides/monorepo-multi-stack": "/project-structure/monorepo-multi-stack",
212 "/guides/monorepo-single-stack": "/project-structure/monorepo-single-stack",
213 "/guides/testing-a-stack": "/testing/testing-a-stack",
214 "/guides/testing-providers": "/testing/testing-providers",
215 "/hetzner/frontend": "/hetzner/frontend/websites",
216 "/hetzner/websites": "/hetzner/frontend/websites",
217 "/neon/branching": "/neon/data/branching",
218 "/neon/connections": "/neon/data/connections",
219 "/neon/migrations": "/neon/data/migrations",
220 "/planetscale/backups": "/planetscale/data/backups",
221 "/planetscale/credentials": "/planetscale/data/credentials",
222 "/planetscale/migrations": "/planetscale/data/migrations",
223 "/planetscale/mysql": "/planetscale/data/mysql",
224 "/planetscale/postgres": "/planetscale/data/postgres",
225 "/railway/frontend": "/railway/frontend/websites",
226 "/railway/websites": "/railway/frontend/websites",
227 "/integrations": "/providers",
228 "/integrations/planetscale": "/planetscale",
229 "/integrations/neon": "/neon",
230 "/integrations/axiom": "/axiom",
231 "/integrations/github": "/github",
232 "/git/auth": "/git/blocks/auth",
233 "/git/storage": "/git/blocks/repositories",
234 "/git/assemblies": "/git/recipes",
235 "/git/assemblies/cloudflare": "/git/recipes/cloudflare",
236 "/git/assemblies/cloudflare-aws": "/git/recipes/cloudflare-aws",
237 "/git/assemblies/one-origin": "/git/tutorial/part-4",
238 "/git/build": "/git/tutorial/part-4",
239 "/git/scale": "/git/recipes/scaling",
240 "/git/scale/pushes": "/git/blocks/hasher",
241 "/git/scale/clones": "/git/blocks/repositories",
242};
243
244const resolveRedirect = (url: URL): string | undefined => {
245 const social = SOCIAL_REDIRECTS[url.pathname.replace(/\/$/, "")];
246 if (social) return social;
247 const reference = referenceDestination(url.pathname + url.search);
248 if (reference) return reference;
249 let p = url.pathname.replace(/\/$/, "");
250 const isMarkdown = p.endsWith(".md");
251 if (isMarkdown) p = p.slice(0, -".md".length);
252 const target = REDIRECTS[p];
253 if (!target) return undefined;
254 if (isMarkdown) return `${target.split("#")[0]}.md`;
255 return target;
256};
257
258export default {
259 fetch: async (request: Request, env: WorkerEnv) => {
260 const url = new URL(request.url);
261 const canonical = url.host === CANONICAL_HOST;
262 const redirect = resolveRedirect(url);
263 if (redirect !== undefined) {
264 return Response.redirect(new URL(redirect, request.url), 301);
265 }
266 if (url.pathname === "/robots.txt" && !canonical) {
267 return new Response(MIRROR_ROBOTS_TXT, {
268 headers: {
269 "content-type": "text/plain; charset=utf-8",
270 "x-robots-tag": "noindex",
271 },
272 });
273 }
274 if (request.method === "GET" && prefersMarkdown(request)) {
275 const mdUrl = toMarkdownUrl(url).toString();
276 const res = await env.ASSETS.fetch(new Request(mdUrl, request));
277 // Astro's asset server labels `.md` as `application/octet-stream`, which
278 // agents treat as a binary download instead of rendering. Force the
279 // correct text type + charset since this branch only ever serves markdown.
280 if (res.status !== 404)
281 return withoutIndexing(
282 withContentType(res, "text/markdown; charset=utf-8"),
283 url,
284 canonical,
285 );
286 }
287 const res = await env.ASSETS.fetch(request);
288 if (res.status === 404 && request.method === "GET") {
289 const v1 = await resolveV1Fallback(url);
290 if (v1 !== undefined) return Response.redirect(v1, 301);
291 }
292 return withoutIndexing(
293 withUtf8Charset(
294 await rewriteAgentTextOrigin(
295 request,
296 rewriteSocialCardHost(request, res),
297 ),
298 ),
299 url,
300 canonical,
301 );
302 },
303};
304
305/**
306 * Served as `robots.txt` on every non-canonical host. Crawlable on purpose:
307 * a `Disallow` would stop crawlers from ever seeing the `noindex` header, and
308 * a blocked URL can still be indexed by URL alone. No `Sitemap:` line, so the
309 * mirror never advertises itself.
310 */
311const MIRROR_ROBOTS_TXT = `# Mirror of https://${CANONICAL_HOST} — every response is noindex.
312User-agent: *
313Allow: /
314`;
315
316/**
317 * Every page is mirrored as raw markdown (`/getting-started.md`, or the same
318 * URL requested with `Accept: text/markdown`) for agents. Search engines crawl
319 * those mirrors too and index them as separate documents — duplicating the HTML
320 * page and surfacing raw MDX (`import { Tabs } from "@astrojs/starlight/..."`)
321 * as the search result's snippet. `llms.txt` / `llms-full.txt` are the same
322 * kind of agent-facing text.
323 *
324 * Keep them crawlable (so the directive is actually seen) but out of the index.
325 * On a non-canonical host *everything* is noindex — the HTML included.
326 */
327const withoutIndexing = (
328 res: Response,
329 url: URL,
330 canonicalHost: boolean,
331): Response => {
332 const ct = res.headers.get("content-type") ?? "";
333 const agentText =
334 url.pathname.endsWith(".md") ||
335 ct.includes("text/markdown") ||
336 ct.includes("text/plain");
337 if (canonicalHost && !agentText) return res;
338 const next = new Response(res.body, res);
339 next.headers.set("x-robots-tag", "noindex");
340 return next;
341};
342
343/**
344 * v1-era URL that still resolves on the v1 docs host, or `undefined`. Only
345 * consulted after production 404s, so the extra HEAD subrequest is confined
346 * to dead links.
347 */
348const resolveV1Fallback = async (url: URL): Promise<string | undefined> => {
349 const pathname = url.pathname.replace(/\/$/, "");
350 if (pathname.endsWith(".md")) return undefined;
351 if (!V1_PREFIXES.some((prefix) => `${pathname}/`.startsWith(prefix))) {
352 return undefined;
353 }
354 const v1Path = pathname.startsWith("/docs/")
355 ? pathname.slice("/docs".length)
356 : pathname;
357 const target = `${V1_ORIGIN}${v1Path}${url.search}`;
358 try {
359 const probe = await fetch(target, { method: "HEAD", redirect: "follow" });
360 return probe.ok ? target : undefined;
361 } catch {
362 return undefined;
363 }
364};
365
366/**
367 * `llms.txt` / `llms-full.txt` are generated at build time with absolute
368 * canonical URLs, so a PR preview would hand agents an index that points back
369 * at production. Rewrite the baked origin to the request's own origin — the
370 * same treatment `rewriteSocialCardHost` gives the OG tags.
371 */
372const AGENT_TEXT_PATHS = new Set(["/llms.txt", "/llms-full.txt"]);
373
374const rewriteAgentTextOrigin = async (
375 request: Request,
376 res: Response,
377): Promise<Response> => {
378 const reqUrl = new URL(request.url);
379 if (
380 !AGENT_TEXT_PATHS.has(reqUrl.pathname) ||
381 reqUrl.host === CANONICAL_HOST ||
382 !res.ok
383 ) {
384 return res;
385 }
386 const body = (await res.text()).replaceAll(
387 `${CANONICAL_ORIGIN}/`,
388 `${reqUrl.origin}/`,
389 );
390 return new Response(body, res);
391};
392
393/**
394 * Astro's static asset server labels `.txt` as `text/plain` with no charset.
395 * UTF-8 bytes (em dashes, arrows in our docs and `llms.txt`) then get decoded
396 * as latin-1 by browsers and agents, showing up as mojibake (`â€"`). Stamp
397 * `charset=utf-8` on text responses that omit it.
398 */
399const withUtf8Charset = (res: Response): Response => {
400 const ct = res.headers.get("content-type");
401 if (!ct || !ct.startsWith("text/") || /charset=/i.test(ct)) return res;
402 return withContentType(res, `${ct}; charset=utf-8`);
403};
404
405const withContentType = (res: Response, contentType: string): Response => {
406 const next = new Response(res.body, res);
407 next.headers.set("content-type", contentType);
408 return next;
409};
410
411/**
412 * Astro bakes absolute URLs into `og:image`, `og:url`, and `twitter:image`
413 * using the `site` config. Rewrite those to the request's host so each
414 * deployment unfurls itself. `<link rel="canonical">` is deliberately NOT
415 * rewritten — it must keep pointing at production on every mirror.
416 */
417const rewriteSocialCardHost = (request: Request, res: Response): Response => {
418 const ct = res.headers.get("content-type") ?? "";
419 if (!ct.includes("text/html")) return res;
420 const reqUrl = new URL(request.url);
421 if (reqUrl.host === CANONICAL_HOST) return res;
422
423 const content = {
424 element(el: HTMLRewriterElement) {
425 const value = el.getAttribute("content");
426 if (!value) return;
427 let u: URL;
428 try {
429 u = new URL(value);
430 } catch {
431 return;
432 }
433 if (u.host !== CANONICAL_HOST) return;
434 u.protocol = reqUrl.protocol;
435 u.host = reqUrl.host;
436 el.setAttribute("content", u.toString());
437 },
438 };
439
440 return new HTMLRewriter()
441 .on('meta[property="og:image"]', content)
442 .on('meta[property="og:url"]', content)
443 .on('meta[name="twitter:image"]', content)
444 .transform(res);
445};
446
447/**
448 * Returns true if the accept header prefers markdown or plain text over HTML.
449 *
450 * Examples:
451 * - opencode - accept: text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, *\/*;q=0.1
452 * - claude code - accept: application/json, text/plain, *\/*
453 *
454 * Notes:
455 * - ChatGPT and Claude web don't set an accept header; maybe check the user agent instead?
456 * - Cursor's headers are too generic (accept: *, user-agent: https://github.com/sindresorhus/got)
457 */
458const prefersMarkdown = (request: Request) => {
459 const accept = request.headers.get("accept");
460 if (!accept) return false;
461
462 // parse accept header and sort by quality; highest quality first
463 const types = accept
464 .split(",")
465 .map((part) => {
466 const type = part.split(";")[0].trim();
467 const q = part.match(/q=([^,]+)/)?.[1];
468 return { type, q: q ? Number.parseFloat(q) : 1 };
469 })
470 .sort((a, b) => b.q - a.q)
471 .map((type) => type.type);
472
473 const markdown = types.indexOf("text/markdown");
474 const plain = types.indexOf("text/plain");
475 const html = types.indexOf("text/html");
476
477 // if no HTML is specified, and either markdown or plain text is specified, prefer markdown
478 if (html === -1) {
479 return markdown !== -1 || plain !== -1;
480 }
481
482 // prefer markdown if higher quality than HTML
483 if ((markdown !== -1 && markdown < html) || (plain !== -1 && plain < html)) {
484 return true;
485 }
486
487 // otherwise, prefer HTML
488 return false;
489};
490
491function toMarkdownUrl(url: URL): URL {
492 const md = new URL(url.toString());
493 let p = md.pathname.replace(/\/$/, "");
494 if (p === "") p = "/index";
495 md.pathname = `${p}.md`;
496 return md;
497}