1#!/usr/bin/env python3
2"""Builds Snowbound for the web (crates/snowbound for wasm32, with crates/snowbound/web) as a
3static folder and deploys it to the VPS, which serves it at https://snowbound.paperclover.net.
4
5 python3 tools/release_web.py --out /tmp/snowbound-web/dist # build only
6 python3 tools/release_web.py --deploy # build, then replace the site
7
8The module and its JavaScript go in a folder named by their contents, b/HASH/, which
9index.html names, so a page never pairs one build's module with another's JavaScript. The
10server is snowbound-site (crates/relay), which pm2's `snowbound-site` runs from the VPS's
11snowbound-web/ on 127.0.0.1:23593, serving site/ and the Live Share codes' pages: it has
12index.html checked on every load and keeps a build's folder for good. Deploying builds it
13for the VPS too, and restarts it only when its binary changed.
14"""
15import argparse
16import gzip
17import hashlib
18import itertools
19import json
20import os
21import re
22from pathlib import Path
23import shutil
24import subprocess
25import sys
26import tempfile
27import time
28import urllib.request
29
30from cargo_fresh import clean_local
31
32ROOT = Path(__file__).resolve().parents[1]
33# The one target folder wasm builds share, apart from the native builds in target/.
34TARGET = ROOT / 'target/wasm'
35WEB = ROOT / 'crates/snowbound/web'
36FONTS = ROOT / 'crates/canvas/assets/fonts'
37# Where the loading shell's icons are looked up by name, in order.
38ICONS = [ROOT / 'crates/snowbound/assets/icons', ROOT / 'crates/canvas/assets/tags', ROOT / 'crates/ui/assets']
39# The VPS's folder: snowbound-site, and the site/ it serves on 127.0.0.1:23593 as pm2's
40# `snowbound-site`, behind https://snowbound.paperclover.net.
41HOST = 'clo@paperclover.net'
42DEPLOYED = 'snowbound-web'
43SERVER = 'snowbound-site'
44URL = 'https://snowbound.paperclover.net'
45# Builds the server keeps beside the newest, for pages still running one.
46KEPT = 3
47EMOJI_URL = 'https://github.com/googlefonts/noto-emoji/raw/v2.051/fonts/Noto-COLRv1.ttf'
48EMOJI_SHA256 = '0ae57fe58645638523ba35f388d93739d292539a9acb84df5700c81b1e1a28d2'
49# As `EMOJI` in src/web.rs names it.
50EMOJI_FONT = 'Noto-COLRv1.ttf.gz'
51# Spelling dictionaries: the name the page asks for, the nixpkgs hunspellDicts package, its
52# files' stem, and the SPDX licences it comes under, each checked to allow redistribution
53# (the LGPL's text with the GPL's, which it amends). Add a row to offer another language.
54DICTIONARIES = [
55 ('en_US', 'en_US', 'en_US', ['BSD-3-Clause']),
56 ('en_GB', 'en_GB-ise', 'en_GB', ['BSD-3-Clause']),
57 ('de_DE', 'de_DE', 'de_DE', ['GPL-2.0-only', 'GPL-3.0-only']),
58 ('fr_FR', 'fr-moderne', 'fr-moderne', ['MPL-2.0']),
59 ('es_ES', 'es_ES', 'es_ES', ['GPL-3.0-only', 'LGPL-3.0-only', 'MPL-1.1']),
60 ('pt_BR', 'pt_BR', 'pt_BR', ['LGPL-3.0-only', 'GPL-3.0-only']),
61 ('pt_PT', 'pt_PT', 'pt_PT', ['GPL-2.0-only', 'LGPL-2.1-only', 'MPL-1.1']),
62 ('it_IT', 'it_IT', 'it_IT', ['GPL-3.0-only']),
63 ('nl_NL', 'nl_NL', 'nl_NL', ['BSD-3-Clause', 'CC-BY-3.0']),
64 ('sv_SE', 'sv_SE', 'sv_SE', ['LGPL-3.0-only', 'GPL-3.0-only']),
65 ('ru_RU', 'ru_RU', 'ru_RU', ['MPL-2.0', 'LGPL-3.0-only', 'GPL-3.0-only']),
66]
67# Size over speed where it costs little: the module is most of the first load.
68PROFILE = ['--config', 'profile.release.opt-level="s"']
69
70
71def run(command, **kwargs):
72 print('+', ' '.join(map(str, command)), flush=True)
73 subprocess.run(command, cwd=ROOT, check=True, **kwargs)
74
75
76def tool(name, nix=None, wasm=False, store=False):
77 """`name` from PATH or Cargo's bin folder, or else from nixpkgs package `nix`; with `wasm`,
78 from nixpkgs only, as the system's clang may not build for wasm32; with `store`, the
79 package's folder."""
80 found = None if wasm or store else shutil.which(name) or shutil.which(name, path=str(Path.home() / '.cargo/bin'))
81 if found:
82 return found
83 if nix and shutil.which('nix'):
84 built = subprocess.check_output(['nix', 'build', f'nixpkgs#{nix}', '--no-link', '--print-out-paths'],
85 text=True).split()[-1]
86 return built if store else str(Path(built) / 'bin' / name)
87 sys.exit(f'{name} is missing: cargo install wasm-bindgen-cli at the version Cargo.lock pins, '
88 'and binaryen for wasm-opt')
89
90
91def environment():
92 """Cargo's environment for wasm32: the wasm target folder, and a clang and llvm-ar that
93 build for wasm32, which SQLite (through sqlite-wasm-rs) needs and Apple's clang lacks."""
94 tools = {}
95 for variable, package, name in (('CC_wasm32_unknown_unknown', 'llvmPackages.clang-unwrapped', 'clang'),
96 ('AR_wasm32_unknown_unknown', 'llvmPackages.llvm', 'llvm-ar')):
97 tools[variable] = os.environ.get(variable) or tool(name, package, wasm=True)
98 return {**os.environ, 'CARGO_TARGET_DIR': str(TARGET), **tools}
99
100
101def build(out):
102 """Writes index.html, the module, its JavaScript and the fonts to `out`, the module knowing
103 when it was built."""
104 built = str(int(time.time()))
105 build_environment = environment()
106 clean_local(ROOT, '--release', '--target', 'wasm32-unknown-unknown', environment=build_environment)
107 run(['cargo', 'build', '--locked', '-p', 'snowbound', '--release', '--target', 'wasm32-unknown-unknown', '--no-default-features', '--features', 'wgpu,live',
108 *PROFILE], env={**build_environment, 'SNOWBOUND_WEB_BUILD': built})
109 if out.exists():
110 shutil.rmtree(out)
111 out.mkdir(parents=True)
112 module = TARGET / 'wasm32-unknown-unknown/release/snowbound.wasm'
113 code = out / 'b/next'
114 run([tool('wasm-bindgen'), '--target', 'web', '--no-typescript', '--out-name', 'snowbound_web',
115 '--out-dir', code, module])
116 bound = code / 'snowbound_web_bg.wasm'
117 run([tool('wasm-opt', 'binaryen'), '-Oz', '--strip-debug', '--strip-producers', bound, '-o', bound])
118 digest = hashlib.sha256()
119 for path in sorted(code.rglob('*')):
120 if path.is_file():
121 digest.update(str(path.relative_to(code)).encode() + b'\0' + path.read_bytes())
122 code = code.rename(out / 'b' / digest.hexdigest()[:16])
123 dictionaries(out / 'dictionaries')
124 (out / 'fonts').mkdir()
125 for font in sorted(FONTS.glob('*')):
126 if font.suffix in ('.ttf', '.txt'):
127 shutil.copy(font, out / 'fonts')
128 fallbacks(out / 'fonts')
129 page(out, code.relative_to(out).as_posix())
130 for path in sorted(out.rglob('*')):
131 if path.is_file():
132 print(f'{path.stat().st_size:>12,} {path.relative_to(out)}')
133
134
135def page(out, code):
136 """Writes index.html with its loading shell's icons inlined, the folder `code` it takes
137 the module and its JavaScript from, and the sizes of the files it fetches before the app
138 starts, for its progress bar."""
139 count = itertools.count()
140
141 def inline(match):
142 name = match[2]
143 svg = next(folder / f'{name}.svg' for folder in ICONS if (folder / f'{name}.svg').exists()).read_text()
144 # Each copy's gradients keep ids of their own in the one document.
145 prefix = f'i{next(count)}-'
146 svg = re.sub(r'id="([^"]+)"', rf'id="{prefix}\1"', svg)
147 svg = re.sub(r'url\(#([^)]+)\)', rf'url(#{prefix}\1)', svg)
148 return f'<i{match[1]}>{svg.strip()}</i>'
149 html = re.sub(r'<i([^>]*) icon="([^"]+)"></i>', inline, (WEB / 'index.html').read_text())
150 sizes = {path.relative_to(out).as_posix(): path.stat().st_size
151 for path in [out / code / 'snowbound_web_bg.wasm', *sorted((out / 'fonts').glob('*.ttf'))]}
152 for marker, written in [('const SIZES = {};', f'const SIZES = {json.dumps(sizes)};'),
153 ('"./snowbound_web.js"', f'"./{code}/snowbound_web.js"'),
154 ('const MODULE = "snowbound_web_bg.wasm";', f'const MODULE = "{code}/snowbound_web_bg.wasm";')]:
155 if marker not in html:
156 sys.exit(f'index.html lacks `{marker}`')
157 html = html.replace(marker, written)
158 (out / 'index.html').write_text(html)
159 shutil.copyfile(ROOT / 'crates/snowbound/assets/icon/sources/Snowbound-Tahoe.icon/Assets/Snowflake.svg',
160 out / 'favicon.svg')
161
162
163def dictionaries(folder):
164 """The Hunspell dictionaries in DICTIONARIES as UTF-8, gzipped for the glue to inflate,
165 each beside its readme, the texts of the licences they come under in licenses/, and their
166 names in index.json for the page to pick from (`pick` in canvas/src/spelling.rs)."""
167 (folder / 'licenses').mkdir(parents=True)
168 texts = Path(tool('spdx', 'spdx-license-list-data.text', store=True)) / 'text'
169 for name, package, stem, licenses in DICTIONARIES:
170 store = Path(tool('hunspell', f'hunspellDicts.{package}', store=True))
171 affix = (store / f'share/hunspell/{stem}.aff').read_bytes()
172 # Hunspell names the files' encoding in the affix file's SET line.
173 encoding = next((line.split()[1] for line in affix.decode('latin-1').splitlines()
174 if line.startswith('SET ')), 'UTF-8')
175 for kind in ('aff', 'dic'):
176 text = (store / f'share/hunspell/{stem}.{kind}').read_bytes().decode(encoding)
177 # One encoding and one line ending for spellbook, which reads only UTF-8. An
178 # 8-bit file's flags are its characters by default, as FLAG UTF-8 keeps them.
179 utf8 = 'SET UTF-8' if encoding == 'UTF-8' or 'FLAG ' in text else 'SET UTF-8\nFLAG UTF-8'
180 text = ''.join((utf8 if line.startswith('SET ') else line) + '\n'
181 for line in text.splitlines())
182 (folder / f'{name}.{kind}.gz').write_bytes(gzip.compress(text.encode(), 9, mtime=0))
183 # Where a package has no readme, its affix file's header comment names the licences.
184 readme = list((store / 'share/doc').glob('*.txt'))
185 notice = readme[0].read_bytes() if readme else \
186 ''.join(line + '\n' for line in affix.decode(encoding).splitlines() if line.startswith('#')).encode()
187 (folder / f'{name}.txt').write_bytes(notice + f'\nLicences: {", ".join(licenses)} (licenses/)\n'.encode())
188 for license in licenses:
189 shutil.copyfile(texts / f'{license}.txt', folder / 'licenses' / f'{license}.txt')
190 (folder / 'index.json').write_text(json.dumps([name for name, *_ in DICTIONARIES]))
191
192
193def fallbacks(fonts):
194 """Noto's faces for scripts the bundled ones lack, which the page fetches as it needs them
195 (`FALLBACKS` in src/web.rs), under the SIL Open Font License."""
196 noto = Path(tool('noto', 'noto-fonts', store=True)) / 'share/fonts/noto'
197 for name in ('NotoSansArabic.ttf', 'NotoSansHebrew.ttf', 'NotoSansDevanagari.ttf', 'NotoSansThai.ttf',
198 'NotoSansSymbols2-Regular.otf'):
199 shutil.copy(noto / name, fonts)
200 # Cutting the CJK face takes a minute or two, so the cut is kept between builds.
201 cjk = TARGET / 'NotoSansCJK.otf'
202 if not cjk.exists():
203 collection = Path(tool('noto', 'noto-fonts-cjk-sans', store=True)) / \
204 'share/fonts/opentype/noto-cjk/NotoSansCJK-VF.otf.ttc'
205 run(['uv', 'run', '--no-project', '--with', 'fonttools', 'python', ROOT / 'tools/web/subset_cjk.py',
206 collection, cjk])
207 shutil.copy(cjk, fonts)
208 # Noto Color Emoji as COLRv1 outlines (4.8 MB), which src/render/colr.rs in draw paints,
209 # gzipped to 2.7 MB for the glue to inflate; nixpkgs has only its 10 MB of bitmaps.
210 emoji = TARGET / EMOJI_FONT
211 if not emoji.exists():
212 data = urllib.request.urlopen(EMOJI_URL).read()
213 if hashlib.sha256(data).hexdigest() != EMOJI_SHA256:
214 sys.exit(f'{EMOJI_URL} is not the font this build pins')
215 emoji.write_bytes(gzip.compress(data, 9, mtime=0))
216 shutil.copy(emoji, fonts)
217 shutil.copy(ROOT / 'crates/draw/assets/NotoColorEmoji-OFL.txt', fonts)
218 (fonts / 'Noto-OFL.txt').write_text('Noto fonts: SIL Open Font License 1.1, https://openfontlicense.org\n')
219
220
221def deploy(built):
222 """Makes the VPS's site `built`, each file taking its place once all have arrived, and
223 leaves the last builds' folders for pages still running them; replaces its server, and
224 restarts it, where the server built now differs."""
225 sync = ['rsync', '--recursive', '--links', '--times', '--compress', '--itemize-changes']
226 arch = subprocess.check_output(['ssh', HOST, 'uname -m'], text=True).strip()
227 with tempfile.TemporaryDirectory() as scratch:
228 run([sys.executable, ROOT / 'tools/release_relay.py', '--architectures', arch, '--output', scratch])
229 server = subprocess.run([*sync, '--checksum', Path(scratch) / f'{SERVER}-linux-{arch}',
230 f'{HOST}:{DEPLOYED}/{SERVER}'],
231 check=True, capture_output=True, text=True).stdout
232 run([*sync, '--delete', '--delay-updates', '--filter=P b/*', f'{built}/', f'{HOST}:{DEPLOYED}/site/'])
233 run(['ssh', HOST, f'cd {DEPLOYED}/site/b && ls -t | tail -n +{KEPT + 2} | xargs -r rm -rf --'])
234 if server.strip():
235 run(['ssh', HOST, f'pm2 restart {SERVER}'])
236 print(f'Deployed {URL}')
237
238
239def main():
240 parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
241 parser.add_argument('--out', type=Path, help='Build into this folder (default: a temporary one)')
242 parser.add_argument('--deploy', action='store_true', help=f'Copy the build to {DEPLOYED}')
243 args = parser.parse_args()
244 if not args.out and not args.deploy:
245 parser.error('give --out, --deploy or both')
246 with tempfile.TemporaryDirectory() as scratch:
247 out = args.out or Path(scratch) / 'web'
248 build(out)
249 if args.deploy:
250 deploy(out)
251
252
253if __name__ == '__main__':
254 main()