1#!/usr/bin/env python3
2import json
3import grp
4import os
5import re
6import subprocess
7import sys
8import time
9import urllib.parse
10import urllib.request
11
12ROUTES = "/var/lib/caddy/routes.caddy"
13TOKEN = "/var/lib/studio/router.token"
14ROUTE_DIR = "/var/lib/studio/routes"
15HOST = re.compile(r"[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?\Z")
16
17
18def nomad(path, token):
19 request = urllib.request.Request(
20 "http://127.0.0.1:4646" + path,
21 headers={"X-Nomad-Token": token},
22 )
23 with urllib.request.urlopen(request, timeout=5) as response:
24 return json.load(response)
25
26
27def passive_health(upstreams, indent):
28 return [f"{indent}fail_duration 30s"] if len(upstreams.split()) > 1 else []
29
30
31def proxy(upstreams, indent, uncompressed=False, host=None):
32 lines = [f"{indent}reverse_proxy {upstreams} {{", f"{indent} lb_try_duration 5s",
33 *passive_health(upstreams, indent + " ")]
34 if uncompressed:
35 lines.append(f"{indent} header_up Accept-Encoding identity")
36 if host:
37 lines.append(f"{indent} header_up Host {host}")
38 return [*lines, f"{indent}}}"]
39
40
41def shale_mcp_routes(port):
42 return [" @shale_mcp_link path_regexp shale_mcp_link ^/-/studio-mcp/([A-Za-z0-9_-]{43})$",
43 " handle @shale_mcp_link {", " rewrite * /oauth/shale/link/{re.shale_mcp_link.1}",
44 " request_header -User-Name", " request_header -User-Groups", " request_header -Studio-Proxy-Token",
45 *proxy(f"127.0.0.1:{port}", " "), " }",
46 " @shale_mcp_callback {", " path /-/callback", " header Cookie *studio_mcp_shale_link=*", " }",
47 " handle @shale_mcp_callback {", " rewrite * /oauth/shale/callback",
48 " request_header -User-Name", " request_header -User-Groups", " request_header -Studio-Proxy-Token",
49 *proxy(f"127.0.0.1:{port}", " "), " }"]
50
51
52def shale_attachment_routes(upstreams, target="/srv/prod/shale"):
53 """Authorize retained attachments through their issue before serving bytes."""
54 manifest = os.path.join(target, "forgejo-attachments.json")
55 if not os.path.exists(manifest):
56 return []
57 with open(manifest) as source:
58 entries = json.load(source)
59 if not isinstance(entries, list) or len(entries) > 10000:
60 raise ValueError("invalid Shale attachment manifest")
61 lines = []
62 for index, entry in enumerate(entries):
63 path, filename, issue = (entry[k] for k in ("path", "file", "issue"))
64 if (not re.fullmatch(r"[0-9a-f-]{36}/[^/\\\"\r\n]+", filename)
65 or path != "/-/forgejo-attachments/" + urllib.parse.quote(filename, safe="/")
66 or not re.fullmatch(r"/(?:[a-zA-Z0-9_.-]+/)+issues/[1-9][0-9]*", issue)):
67 raise ValueError("unsafe Shale attachment route")
68 # Request cookies go to Shale as usual. A private issue yields 403/404;
69 # only Shale's successful issue response permits the local file read.
70 name = f"@shale_attachment_{index}"
71 lines += [f" {name} path {json.dumps(path)}", f" handle {name} {{",
72 " header Cache-Control private,no-store", " header X-Content-Type-Options nosniff",
73 f" reverse_proxy {upstreams} {{", " method GET", f" rewrite {issue}",
74 " @authorized status 200", " handle_response @authorized {",
75 f" root * {json.dumps(os.path.join(target, 'forgejo-attachments'))}",
76 f" rewrite * {json.dumps('/' + urllib.parse.quote(filename, safe='/'))}",
77 " file_server", " }", " }", " }"]
78 return lines
79
80
81def shale_write_routes(host):
82 # Older Shale releases predate form tokens. Browsers send Origin on POST;
83 # require the exact site origin for every cookie-authenticated mutation.
84 # HTTP Git clients use Basic authentication and do not carry this cookie.
85 return [" @shale_unsafe_origin {", " method POST PUT PATCH DELETE",
86 " header Cookie *SessionID=*", f" not header Origin https://{host}",
87 " }", ' respond @shale_unsafe_origin "Forbidden" 403']
88
89
90def shale_git_routes(upstreams):
91 lines = [" @shale_git path */info/refs */git-upload-pack */git-receive-pack",
92 " handle @shale_git {", f" reverse_proxy {upstreams} {{"]
93 # Shale forwards CGI Status as a header instead of the HTTP status.
94 for status in (400, 403, 404, 405, 415, 500):
95 lines += [f' @cgi_{status} header Status "{status} *"',
96 f" handle_response @cgi_{status} {{",
97 " header {", " -Status", " defer", " }",
98 f" copy_response {status}", " }"]
99 return [*lines, " }", " }"]
100
101
102def render(token):
103 with open(os.environ["STUDIO_PROXY_TOKEN_FILE"]) as file:
104 dashboard_proof = file.read().strip()
105 if not re.fullmatch(r"[0-9a-fA-F]{64}", dashboard_proof):
106 raise ValueError("invalid dashboard proxy token")
107 auth_host = "auth." + os.environ["STUDIO_DOMAIN"]
108 legacy_host = "keycloak." + os.environ["STUDIO_DOMAIN"]
109 origins = json.dumps({"origins": ["https://snowglobe." + os.environ["STUDIO_DOMAIN"]]}, separators=(",", ":"))
110 webauthn = [" handle /.well-known/webauthn {", ' header Content-Type application/json',
111 f" respond {json.dumps(origins)} 200", " }"]
112 routes = {}
113 internal_services = {}
114 auth_upstreams = set()
115 allocation_checks = {}
116 for namespace in nomad("/v1/services", token):
117 if namespace["Namespace"] != "default":
118 continue
119 for service in namespace["Services"]:
120 name = urllib.parse.quote(service["ServiceName"], safe="")
121 for instance in nomad(f"/v1/service/{name}", token):
122 tags = instance.get("Tags") or []
123 if instance["ServiceName"] != "forward-auth" and not any(tag.startswith("caddy-") for tag in tags):
124 continue
125 address = instance["Address"]
126 port = instance["Port"]
127 if not re.fullmatch(r"[0-9a-fA-F:.]+", address) or type(port) is not int or not 1 <= port <= 65535:
128 continue
129 upstream = f"[{address}]:{port}" if ":" in address else f"{address}:{port}"
130 if not re.fullmatch(r"[a-z][a-z0-9-]*", instance["ServiceName"]):
131 raise ValueError("invalid internal service name")
132 internal_services.setdefault(instance["ServiceName"], set()).add(upstream)
133 alloc = instance["AllocID"]
134 if alloc not in allocation_checks:
135 allocation_checks[alloc] = nomad(f"/v1/allocation/{alloc}/checks", token)
136 service_checks = list(allocation_checks[alloc].values())
137 if not service_checks or any(check["Status"] != "success" for check in service_checks):
138 continue
139 if instance["ServiceName"] == "forward-auth":
140 auth_upstreams.add(upstream)
141 access = [tag.split("=", 1)[1] for tag in tags if tag.startswith("caddy-auth-role=")]
142 user_headers = [tag.split("=", 1)[1] for tag in tags if tag.startswith("caddy-user-header=")]
143 if len(access) > 1 or (access and not re.fullmatch(r"[a-z][a-z0-9-]*", access[0])):
144 raise ValueError("invalid route access role")
145 if len(user_headers) > 1 or (user_headers and (not access or not re.fullmatch(r"[A-Za-z][A-Za-z0-9-]*", user_headers[0]))):
146 raise ValueError("invalid authenticated user header")
147 if access and not any(tag.startswith("caddy-host=") for tag in tags):
148 raise ValueError("route access role has no host")
149 for tag in tags:
150 if tag.startswith("caddy-host="):
151 host = tag.split("=", 1)[1]
152 if not HOST.fullmatch(host):
153 raise ValueError(f"invalid Caddy host: {host!r}")
154 route = routes.setdefault((80, host), {"upstreams": set(), "services": set(), "authRole": None, "userHeader": None, "internal": False, "realIp": False})
155 if access and route["authRole"] not in (None, access[0]):
156 raise ValueError("conflicting route access roles")
157 if user_headers and route["userHeader"] not in (None, user_headers[0]):
158 raise ValueError("conflicting authenticated user headers")
159 if access:
160 route["authRole"] = access[0]
161 if user_headers:
162 route["userHeader"] = user_headers[0]
163 if "caddy-internal=true" in tags:
164 route["internal"] = True
165 if "caddy-real-ip=true" in tags:
166 route["realIp"] = True
167 route["upstreams"].add(upstream)
168 route["services"].add(instance.get("JobID") or instance["ServiceName"])
169 elif tag.startswith("caddy-port="):
170 listener = int(tag.split("=", 1)[1])
171 if not 1 <= listener <= 65535:
172 raise ValueError(f"invalid Caddy port: {listener}")
173 route = routes.setdefault((listener, ""), {"upstreams": set(), "services": set()})
174 route["upstreams"].add(upstream)
175
176 lines = []
177 for (listener, host), route in sorted(routes.items()):
178 if host:
179 if host in {auth_host, legacy_host}:
180 continue
181 role = route["authRole"]
182 if role and not auth_upstreams:
183 continue
184 lines.append(f"{host if listener == 80 else f'{host}:{listener}'} {{")
185 if host.endswith(".test") or route["internal"]:
186 lines.append(" tls internal")
187 if route["realIp"]:
188 lines.append(" request_header X-Real-Ip {remote_host}")
189 if len(route["services"]) != 1:
190 raise ValueError(f"multiple services claim {host}")
191 service = next(iter(route["services"]))
192 if not re.fullmatch(r"[a-z][a-z0-9-]*", service):
193 raise ValueError(f"invalid service name: {service}")
194 if service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service):
195 # Keep the Origin guard before every static/proxy handle;
196 # otherwise Caddy sorts those handles ahead of respond.
197 lines += [" route {", *shale_write_routes(host),
198 *shale_git_routes(" ".join(sorted(route["upstreams"])))]
199 if re.fullmatch(r"shale-preview-[0-9a-f]{8}", service):
200 lines += shale_attachment_routes(" ".join(sorted(route["upstreams"])),
201 "/srv/staging/" + service)
202 if service == "shale":
203 lines += [" handle /-/logout {", f" reverse_proxy {' '.join(sorted(route['upstreams']))} {{",
204 f" header_down Location https://snowglobe.{os.environ['STUDIO_DOMAIN']}/auth/shale/sign-out",
205 " @logged_out status 200 302 303 404", " handle_response @logged_out {",
206 " copy_response 303", " }", " }", " }"]
207 lines += shale_attachment_routes(" ".join(sorted(route["upstreams"])))
208 lines += [" @userscript path_regexp userscript ^/userscripts/discord-pluralkit-predict(/.*)?$",
209 " redir @userscript /discord-pluralkit-predict{re.userscript.1}?{query} 308"]
210 port = int(os.environ["STUDIO_DASHBOARD_PORT"])
211 if not 1 <= port <= 65535:
212 raise ValueError("invalid dashboard port for Shale linking")
213 lines += shale_mcp_routes(port)
214 lines += [" tracing {", f" span {service}", " span_attributes {",
215 f" studio.service {service}", " studio.kind edge", " }", " }"]
216 manifest = os.path.join(ROUTE_DIR, service + ".json")
217 assets = {}
218 headers = {}
219 head_html = {}
220 proof = None
221 scrub = []
222 if os.path.exists(manifest):
223 with open(manifest) as file:
224 assets = json.load(file)
225 identity = assets.get("identity", {})
226 headers = identity.get(host, {})
227 head_html = assets.get("headHtml", {}).get(host, {})
228 proof = assets.get("identityProof", {}).get(host)
229 scrub = sorted({name for configured in identity.values() for name in configured} | set(assets.get("identityProof", {}).values()))
230 if any(not re.fullmatch(r"[A-Za-z][A-Za-z0-9-]*", name) for name in scrub):
231 raise ValueError(f"invalid identity header: {host}")
232 if proof and not headers:
233 raise ValueError(f"identity proof has no identity headers: {host}")
234 if any(source not in {"X-Auth-Request-User", "X-Auth-Request-Groups", "X-Auth-Request-Preferred-Username"} for source in headers.values()):
235 raise ValueError(f"invalid identity claim: {host}")
236 if role and headers:
237 raise ValueError(f"route has both required and optional auth: {host}")
238 if role and (assets.get("files") or assets.get("dirs")):
239 raise ValueError(f"authenticated route has static overrides: {host}")
240 if head_html and (role or headers):
241 raise ValueError(f"authenticated route has HTML injection: {host}")
242 for request, markup in head_html.items():
243 if not re.fullmatch(r"/[A-Za-z0-9/._*-]*", request) or not markup:
244 raise ValueError(f"invalid HTML injection: {host} {request}")
245 for request, path in assets.get("files", {}).items():
246 if not request.startswith("/") or " " in request or "\n" in request:
247 raise ValueError(f"invalid static path: {request}")
248 lines += [f" handle {request} {{", f" root * {json.dumps(os.path.dirname(path))}",
249 f" rewrite * /{os.path.basename(path)}", ' header Cache-Control "no-store"',
250 " file_server", " }"]
251 for request, path in assets.get("dirs", {}).items():
252 if not request.startswith("/") or " " in request or "\n" in request:
253 raise ValueError(f"invalid static prefix: {request}")
254 lines += [f" handle_path {request}* {{", f" root * {json.dumps(path)}",
255 " file_server", " }"]
256 metrics_path = assets.get("metricsPaths", {}).get(host)
257 if metrics_path:
258 if not re.fullmatch(r"/[A-Za-z0-9/._-]+", metrics_path):
259 raise ValueError(f"invalid metrics path: {host}")
260 lines += [f" handle {metrics_path} {{", " respond 404", " }"]
261 upstreams = " ".join(sorted(route["upstreams"]))
262 if role:
263 auth = " ".join(sorted(auth_upstreams))
264 lines += [" handle /snow.oauth2/* {", *proxy(auth, " "), " }", " handle {"]
265 if route["userHeader"]:
266 lines.append(f" request_header -{route['userHeader']}")
267 lines += [
268 f" reverse_proxy {auth} {{", " lb_try_duration 5s", *passive_health(auth, " "), " method GET",
269 " rewrite /snow.oauth2/auth", " header_up X-Forwarded-Method {method}",
270 " header_up X-Forwarded-Uri {uri}", " @unauthorized status 401",
271 " handle_response @unauthorized {",
272 " redir * /snow.oauth2/sign_in?rd={scheme}://{host}{uri}", " }",
273 f" @allowed header X-Auth-Request-Groups *role:{role}*",
274 " handle_response @allowed {", " method {method}", " rewrite {uri}",
275 ]
276 if route["userHeader"]:
277 lines.append(f" request_header {route['userHeader']} {{rp.header.X-Auth-Request-Preferred-Username}}")
278 lines += [
279 *proxy(upstreams, " "), " }",
280 " handle_response {", " respond 403", " }", " }", " }", "}",
281 ]
282 elif headers and (auth_upstreams or service == "copyparty"):
283 native = service == "copyparty"
284 auth = f"127.0.0.1:{int(os.environ['STUDIO_DASHBOARD_PORT'])}" if native else " ".join(sorted(auth_upstreams))
285 if native:
286 lines += [" handle /auth/file/* {", " request_header -User-Name", " request_header -User-Groups",
287 f" request_header Studio-Proxy-Token {dashboard_proof}",
288 " request_header X-Studio-Client-IP {remote_host}", *proxy(auth," "), " }"]
289 else:
290 lines += [" handle /snow.oauth2/* {", *proxy(auth," "), " }"]
291 lines += [" handle {"]
292 lines += [f" request_header -{name}" for name in scrub]
293 lines += [f" reverse_proxy {auth} {{", " lb_try_duration 5s", *passive_health(auth, " "), " method GET", " rewrite " + ("/auth/file/check" if native else "/snow.oauth2/auth"),
294 *([f" header_up Studio-Proxy-Token {dashboard_proof}"] if native else []),
295 " header_up X-Forwarded-Method {method}", " header_up X-Forwarded-Uri {uri}",
296 " @authenticated status 2xx", " handle_response @authenticated {"]
297 lines += [f" request_header {name} {{rp.header.{source}}}" for name, source in headers.items()]
298 if proof:
299 lines.append(f" request_header {proof} 1")
300 lines += [" }", " @anonymous status 4xx", " handle_response @anonymous {",
301 f" request_header -{scrub[0]}", " }", " }",
302 *proxy(upstreams, " "), " }", "}"]
303 else:
304 page_upstream = upstreams
305 shale_page = service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service)
306 page_headers = []
307 if shale_page:
308 page_upstream = f"127.0.0.1:{int(os.environ['STUDIO_DASHBOARD_PORT'])}"
309 page_headers = [f" request_header Studio-Proxy-Token {dashboard_proof}",
310 f" request_header Studio-Shale-Upstream {json.dumps(sorted(route['upstreams'])[0])}",
311 " request_header Studio-Shale-Uri {uri}",
312 " rewrite * /auth/shale/page"]
313 for index, (request, markup) in enumerate(head_html.items()):
314 name = f"@studio_head_{index}"
315 lines += [f" {name} path {request}", f" handle {name} {{", " route {",
316 f" replace </head> {json.dumps(markup + '</head>')} {{",
317 " match {", " header Content-Type text/html*", " }", " }",
318 *[" " + line for line in page_headers],
319 *proxy(page_upstream, " ", uncompressed=True), " }", " }"]
320 page_handler = [*page_headers, *proxy(page_upstream, " ")]
321 if shale_page:
322 page_handler = [" route {", *[" " + line for line in page_handler], " }"]
323 lines += [" handle {", *(f" request_header -{name}" for name in scrub),
324 *page_handler, " }", "}"]
325 if service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service):
326 lines.insert(len(lines) - 1, " }")
327 else:
328 lines += [f":{listener} {{", *proxy(" ".join(sorted(route["upstreams"])), " "), "}"]
329 dashboard_host = "snowglobe." + os.environ["STUDIO_DOMAIN"]
330 if any(not HOST.fullmatch(host) for host in [auth_host, legacy_host, dashboard_host]):
331 raise ValueError("invalid login domain")
332 for host in [auth_host, legacy_host]:
333 lines += [f"{host} {{", *([" tls internal"] if host.endswith(".test") else []),
334 *(webauthn if host == auth_host else []), " handle {",
335 f" redir https://{dashboard_host}/ 308", " }", "}"]
336 traces = next((route for route in routes.values() if "victoria-traces" in route["services"]), None)
337 if traces:
338 lines += ["http://127.0.0.1:10428 {", " bind 127.0.0.1",
339 *proxy(" ".join(sorted(traces["upstreams"])), " "), "}"]
340 dashboard_port = int(os.environ["STUDIO_DASHBOARD_PORT"])
341 if not 1 <= dashboard_port <= 65535:
342 raise ValueError("invalid dashboard route")
343 if (80, dashboard_host) in routes:
344 raise ValueError(f"dashboard route conflicts with a Nomad service: {dashboard_host}")
345 lines.append(f"{dashboard_host} {{")
346 if dashboard_host.endswith(".test"):
347 lines.append(" tls internal")
348 lines += [" encode zstd gzip", " tracing {", " span globe", " span_attributes {", " studio.kind edge", " }", " }"]
349 lines += [" handle {", " request_header -User-Name", " request_header -User-Groups",
350 f" request_header Studio-Proxy-Token {dashboard_proof}",
351 " request_header X-Studio-Client-IP {remote_host}",
352 *proxy(f"127.0.0.1:{dashboard_port}", " "), " }", "}"]
353 internal_port = os.environ.get("STUDIO_INTERNAL_PORT")
354 if internal_port is not None:
355 internal_host = "dashboard.internal." + os.environ["STUDIO_DOMAIN"]
356 if not HOST.fullmatch(internal_host) or not internal_port.isdecimal() or not 1 <= int(internal_port) <= 65535:
357 raise ValueError("invalid internal dashboard route")
358 if (80, internal_host) in routes or (int(internal_port), "") in routes:
359 raise ValueError("internal dashboard route conflicts with a Nomad service")
360 lines += [f"{internal_host}:{internal_port} {{", " tls internal",
361 f" @dashboard header Studio-Proxy-Token {dashboard_proof}",
362 " handle @dashboard {", " request_header -Studio-Proxy-Token",
363 " request_header -User-Name", " request_header -User-Groups",
364 " handle_path /nomad/* {", *proxy("127.0.0.1:4646", " ", host="{upstream_hostport}"), " }"]
365 for service, upstreams in sorted(internal_services.items()):
366 host = f"{service}.{os.environ['STUDIO_DOMAIN']}" if service == "shale" or re.fullmatch(r"shale-preview-[0-9a-f]{8}", service) else "{upstream_hostport}"
367 lines += [f" handle_path /services/{service}/* {{",
368 *proxy(" ".join(sorted(upstreams)), " ", host=host), " }"]
369 lines += [" handle {", " respond 404", " }", " }", " handle {", " respond 403", " }", "}"]
370 return "\n".join(lines) + "\n"
371
372
373def update(content):
374 try:
375 with open(ROUTES) as file:
376 previous = file.read()
377 except FileNotFoundError:
378 previous = ""
379 if content == previous:
380 return
381 pending = ROUTES + ".pending"
382 def replace(value):
383 with open(pending, "w") as file:
384 file.write(value)
385 os.chown(pending, -1, grp.getgrnam("caddy").gr_gid)
386 os.chmod(pending, 0o640)
387 os.replace(pending, ROUTES)
388
389 replace(content)
390 result = subprocess.run(["systemctl", "reload", "caddy"], capture_output=True, text=True)
391 if result.returncode:
392 replace(previous)
393 raise RuntimeError(result.stderr.strip())
394 print("Caddy routes updated", flush=True)
395
396
397def main():
398 with open(TOKEN) as file:
399 token = file.read().strip()
400 while True:
401 try:
402 update(render(token))
403 except Exception as error:
404 print(f"Caddy route update failed: {error}", file=sys.stderr, flush=True)
405 time.sleep(5)
406
407
408if __name__ == "__main__":
409 main()