1#!/usr/bin/env python3
2import argparse
3from contextlib import ExitStack
4from html.parser import HTMLParser
5from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
6import json
7import os
8from pathlib import Path
9import socket
10import sqlite3
11import subprocess
12import tempfile
13import threading
14import time
15import urllib.error
16import urllib.request
17
18import router
19
20
21class Links(HTMLParser):
22 def __init__(self, html):
23 super().__init__()
24 self.links = []
25 self.feed(html)
26
27 def handle_starttag(self, tag, attrs):
28 if tag == 'a':
29 self.links.append(dict(attrs))
30
31
32def port():
33 with socket.socket() as sock:
34 sock.bind(('127.0.0.1', 0))
35 return sock.getsockname()[1]
36
37
38def main():
39 parser = argparse.ArgumentParser()
40 parser.add_argument('--caddy', required=True, type=Path)
41 parser.add_argument('--binary', type=Path, default=Path(__file__).resolve().parent.parent / 'dashboard/target/debug/home-dashboard')
42 parser.add_argument('--page', type=Path)
43 parser.add_argument('--browser-ready-file', type=Path)
44 args = parser.parse_args()
45 repo = Path(__file__).resolve().parent.parent
46 observations = []
47 issue = args.page.read_bytes() if args.page else b'<html><head></head><body><a href="/~guest-github-24465214"><img src="avatar"><span>~guest-github-24465214</span></a></body></html>'
48 sample = b'<a href="/~guest-astheno-abc">guest</a><a href="/~guest-github-unknown">unknown</a><a href="/~guest-github-24465214/repo">repo</a>'
49 blob = bytes(range(256)) * 65536
50
51 class Fixture(BaseHTTPRequestHandler):
52 protocol_version = 'HTTP/1.1'
53
54 def do_GET(self):
55 observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}))
56 if self.path == '/binary':
57 return self.respond(blob, 'application/octet-stream')
58 if self.path == '/large':
59 return self.respond(b' ' * (9 * 1024 * 1024), 'text/html')
60 if self.path == '/-/login':
61 callback = 'https://' + self.headers['Host'] + '/-/callback'
62 return self.respond(b'', 'text/plain', 302, [('Location', 'https://snowglobe.studio.test/auth/oidc/authorize?redirect_uri=' + urllib.parse.quote(callback, safe=''))])
63 if self.path == '/redirect':
64 return self.respond(b'', 'text/plain', 302, [('Location', '/snowbound/issues/26'), ('Set-Cookie', 'SessionID=new; Path=/; HttpOnly'), ('Set-Cookie', 'other=kept; Path=/')])
65 if self.path.startswith('/-/') and args.page:
66 with urllib.request.urlopen('https://shale.paperclover.net' + self.path, timeout=15) as response:
67 return self.respond(response.read(), response.headers['Content-Type'])
68 return self.respond(issue + sample, 'text/html; charset=utf-8', headers=[('ETag', '"stale"'), ('Last-Modified', 'Mon, 05 Oct 2026 00:00:00 GMT')])
69
70 def do_HEAD(self):
71 self.respond(b'', 'text/html')
72
73 def do_POST(self):
74 body = self.rfile.read(int(self.headers.get('Content-Length', 0)))
75 observations.append((self.command, self.path, {key.lower(): value for key, value in self.headers.items()}, body))
76 self.respond(body, 'application/octet-stream', 201)
77
78 def respond(self, body, content_type, status=200, headers=()):
79 self.send_response(status)
80 self.send_header('Content-Type', content_type)
81 self.send_header('Content-Length', str(len(body)))
82 for key, value in headers:
83 self.send_header(key, value)
84 self.end_headers()
85 self.wfile.write(body)
86
87 def log_message(self, *args):
88 pass
89
90 class NoRedirect(urllib.request.HTTPRedirectHandler):
91 def redirect_request(self, *args):
92 return None
93
94 opener = urllib.request.build_opener(NoRedirect)
95 with tempfile.TemporaryDirectory(prefix='shale-page-') as temporary, ExitStack() as stack:
96 root = Path(temporary).resolve()
97 fixture = ThreadingHTTPServer(('127.0.0.1', 0), Fixture)
98 stack.callback(fixture.server_close)
99 stack.callback(fixture.shutdown)
100 threading.Thread(target=fixture.serve_forever, daemon=True).start()
101 dashboard_port, gateway_port, internal_port = port(), port(), port()
102 proof = 'a' * 64
103 token = root / 'proxy.token'
104 token.write_text(proof)
105 certificate, gateway_key = root / 'gateway.pem', root / 'gateway.key'
106 subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost', '-addext', 'basicConstraints=CA:FALSE', '-keyout', str(gateway_key), '-out', str(certificate)], check=True, capture_output=True)
107 data = root / 'data'
108 data.mkdir()
109 environment = {**os.environ, 'PORT': str(dashboard_port), 'STUDIO_DOMAIN': 'studio.test', 'STUDIO_DATA_DIR': str(data), 'STUDIO_PROXY_TOKEN_FILE': str(token), 'STUDIO_INTERNAL_URL': f'https://localhost:{internal_port}', 'STUDIO_CA_BUNDLE': str(certificate), 'STUDIO_REPO': str(repo), 'STUDIO_WEB_DIR': str(repo / 'dashboard/dist')}
110 for key in ['STUDIO_INDEX_POOL', 'STUDIO_AUTH_REQUIRED', 'STUDIO_YT_STATE']:
111 environment.pop(key, None)
112 log = stack.enter_context((root / 'dashboard.log').open('w'))
113 dashboard = subprocess.Popen([str(args.binary)], env=environment, stdout=log, stderr=log)
114 stack.callback(lambda: dashboard.wait(timeout=10))
115 stack.callback(dashboard.terminate)
116 deadline = time.monotonic() + 15
117 while True:
118 assert dashboard.poll() is None, (root / 'dashboard.log').read_text()
119 try:
120 with socket.create_connection(('127.0.0.1', dashboard_port), timeout=.1):
121 break
122 except OSError:
123 assert time.monotonic() < deadline
124 time.sleep(.05)
125 with sqlite3.connect(data / 'accounts.sqlite') as db:
126 for provider, subject, suffix, name in [('github', '24465214', '24465214', 'paperclover'), ('astheno', 'pairwise-astheno-subject', 'abc', 'Astheno user'), ('github', '777', '777', None), ('astheno', 'unmapped-pairwise-subject', 'unknown', 'Unmapped guest')]:
127 username = f'guest-{provider}-{suffix}'
128 profile = {'kind': 'guest', 'enabled': True, 'username': username, 'guestProvider': provider, 'firstName': name}
129 db.execute('INSERT INTO users(id,profile) VALUES (?,?)', [username, json.dumps(profile)])
130 db.execute('INSERT INTO external_identities VALUES (?,?,?)', [provider, subject, username])
131 os.environ.update(STUDIO_DOMAIN='studio.test', STUDIO_DASHBOARD_PORT=str(dashboard_port), STUDIO_PROXY_TOKEN_FILE=str(token), STUDIO_INTERNAL_PORT=str(internal_port))
132 router.ROUTE_DIR = str(root / 'routes')
133 Path(router.ROUTE_DIR).mkdir()
134 (Path(router.ROUTE_DIR) / 'shale.json').write_text(json.dumps({'headHtml': {'shale.studio.test': {'/snowbound/*': '<meta name="rewrite-test" content="kept">'}}}))
135
136 def nomad(path, token):
137 if path == '/v1/services':
138 return [{'Namespace': 'default', 'Services': [{'ServiceName': 'shale'}, {'ServiceName': 'shale-preview-12345678'}]}]
139 if path.startswith('/v1/service/'):
140 service = path.rsplit('/', 1)[1]
141 host = 'shale.studio.test' if service == 'shale' else service + '.studio.test'
142 return [{'Address': '127.0.0.1', 'Port': fixture.server_port, 'ServiceName': service, 'AllocID': service, 'JobID': service, 'Tags': ['caddy-host=' + host]}]
143 return {'ready': {'Status': 'success'}}
144
145 router.nomad = nomad
146 rendered = router.render('fixture')
147 full_config = root / 'routes.caddy'
148 full_config.write_text('{\n admin off\n auto_https off\n}\n' + rendered)
149 caddy_env = {**os.environ, 'XDG_DATA_HOME': str(root / 'caddy-data'), 'XDG_CONFIG_HOME': str(root / 'caddy-config')}
150 subprocess.run([str(args.caddy), 'validate', '--config', str(full_config), '--adapter', 'caddyfile'], check=True, capture_output=True, env=caddy_env)
151 def local_site(host, listener):
152 start = rendered.index(host + ' {')
153 end = rendered.index('\n}', start) + 2
154 return rendered[start:end].replace(host + ' {', f'http://127.0.0.1:{listener} {{', 1).replace(' tls internal\n', '').replace(' route {\n', f' route {{\n request_header Host {host}\n', 1)
155
156 preview_port = port()
157 config = '{\n admin off\n auto_https off\n}\n' + local_site('shale.studio.test', gateway_port) + '\n' + local_site('shale-preview-12345678.studio.test', preview_port)
158 internal_host = f'dashboard.internal.studio.test:{internal_port}'
159 start = rendered.index(internal_host + ' {')
160 config += '\n' + rendered[start:].replace(internal_host + ' {', f'https://localhost:{internal_port} {{', 1).replace(' tls internal\n', f' tls {certificate} {gateway_key}\n', 1)
161 config_path = root / 'Caddyfile'
162 config_path.write_text(config)
163 subprocess.run([str(args.caddy), 'validate', '--config', str(config_path), '--adapter', 'caddyfile'], check=True, capture_output=True, env=caddy_env)
164 caddy_log = stack.enter_context((root / 'caddy.log').open('w'))
165 caddy = subprocess.Popen([str(args.caddy), 'run', '--config', str(config_path), '--adapter', 'caddyfile'], stdout=caddy_log, stderr=caddy_log, env=caddy_env)
166 stack.callback(lambda: caddy.wait(timeout=10))
167 stack.callback(caddy.terminate)
168 origin = f'http://127.0.0.1:{gateway_port}'
169
170 def request(path, *, headers=None, body=None, direct=False):
171 supplied = {'Cookie': 'SessionID=preserved', 'Origin': 'https://shale.studio.test', **(headers or {})}
172 url = (f'http://127.0.0.1:{dashboard_port}' if direct else origin) + path
173 try:
174 response = opener.open(urllib.request.Request(url, headers=supplied, data=body), timeout=15)
175 except urllib.error.HTTPError as error:
176 response = error
177 with response:
178 return response.status, response.headers, response.read()
179
180 deadline = time.monotonic() + 15
181 while True:
182 try:
183 status, headers, body = request('/snowbound/issues/26?query=kept')
184 break
185 except urllib.error.URLError:
186 assert time.monotonic() < deadline and caddy.poll() is None
187 time.sleep(.05)
188 assert status == 200, (status, body, (root / 'dashboard.log').read_text(), (root / 'caddy.log').read_text())
189 links = Links(body.decode()).links
190 link = next(link for link in links if link.get('href') == 'https://github.com/paperclover')
191 assert link['target'] == '_blank' and link['rel'] == 'noreferrer', link
192 assert any(link.get('href') == '/~guest-github-unknown' for link in links)
193 assert any(link.get('href') == '/~guest-github-24465214/repo' for link in links)
194 assert b'Astheno user' in body and not any(link.get('href') == '/~guest-astheno-abc' for link in links)
195 assert b'identity.astheno.software/user/' not in body and b'pairwise-astheno-subject' not in body
196 assert b'https://avatars.githubusercontent.com/u/24465214?s=64' in body
197 assert b'rewrite-test' in body
198 assert headers['Cache-Control'] == 'no-store' and headers.get('ETag') is None and headers.get('Last-Modified') is None
199 assert observations[-1][1] == '/snowbound/issues/26?query=kept', observations[-1]
200 assert observations[-1][2]['host'] == 'shale.studio.test', observations[-1]
201 assert observations[-1][2]['cookie'] == 'SessionID=preserved', observations[-1]
202 assert not any(key.lower().startswith('studio-') for key in observations[-1][2]), observations[-1]
203 status, headers, body = request('/~guest-github-24465214')
204 assert status == 302 and headers['Location'] == 'https://github.com/paperclover' and headers['Referrer-Policy'] == 'no-referrer'
205 assert request('/~guest-astheno-abc/')[0] == 200
206 assert observations[-1][1] == '/~guest-astheno-abc/'
207 assert request('/~guest-astheno-unknown')[0] == 200
208 status, headers, body = request('/-/login')
209 assert status == 302 and urllib.parse.parse_qs(urllib.parse.urlparse(headers['Location']).query)['redirect_uri'] == ['https://shale.studio.test/-/callback']
210 status, headers, body = request('/redirect')
211 assert status == 302 and headers['Location'] == '/snowbound/issues/26' and len(headers.get_all('Set-Cookie')) == 2
212 with opener.open(f'http://127.0.0.1:{preview_port}/snowbound/issues/26') as response:
213 assert response.status == 200
214 assert observations[-1][2]['host'] == 'shale-preview-12345678.studio.test'
215 assert any(link.get('href') == 'https://github.com/paperclover' for link in Links(response.read().decode()).links)
216 assert b'/~guest-github-24465214' in request('/repo/info/refs')[2]
217 assert request('/binary')[2] == blob
218 assert len(request('/large')[2]) == 9 * 1024 * 1024
219 status, headers, body = request('/submit?keep=yes', body=b'field=unchanged')
220 assert status == 201 and body == b'field=unchanged' and observations[-1][1] == '/submit?keep=yes'
221 assert request('/submit', body=b'blocked', headers={'Origin': 'https://other.example'})[0] == 403
222 assert request('/auth/shale/page', direct=True)[0] == 403
223 assert request('/auth/shale/page', direct=True, headers={'Studio-Proxy-Token': proof, 'Studio-Shale-Upstream': f'127.0.0.1:{fixture.server_port}', 'Studio-Shale-Uri': '/', 'Host': 'snowglobe.studio.test'})[0] == 403
224 print(json.dumps({'html_links': 'passed', 'provider_redirects': 'passed', 'cookies_forms_and_binary': 'passed', 'proxy_boundary': 'passed', 'head_injection': 'passed', 'origin': origin}), flush=True)
225 if args.browser_ready_file:
226 args.browser_ready_file.write_text(json.dumps({'origin': origin}))
227 stop = args.browser_ready_file.with_suffix('.stop')
228 deadline = time.monotonic() + 600
229 while not stop.exists() and time.monotonic() < deadline:
230 time.sleep(.2)
231 args.browser_ready_file.unlink(missing_ok=True)
232 stop.unlink(missing_ok=True)
233
234
235if __name__ == '__main__':
236 main()