1#!/usr/bin/env python3
2"""Builds, signs and publishes Snowbound for each desktop platform; see tools/RELEASE.md."""
3import argparse
4import base64
5from datetime import datetime
6import hashlib
7import json
8import os
9from pathlib import Path
10import re
11import shutil
12import subprocess
13import sys
14import tempfile
15import time
16import zipfile
17from zoneinfo import ZoneInfo
18
19from cargo_fresh import clean_local
20
21ROOT = Path(__file__).resolve().parents[1]
22PUBLISHED = Path('/Volumes/clover/Documents/Public/Snowbound')
23URL = 'https://file.paperclover.net/shr/snowbound/'
24KEY = Path.home() / '.config/snowbound/release-key'
25# Its public half, which the app checks updates against.
26MINISIGN = ROOT / 'minisign.pub'
27ZONE = ZoneInfo('America/Los_Angeles')
28PLATFORMS = ['macos-aarch64', 'macos-x86_64', 'macos-10.6', 'linux-x86_64', 'linux-aarch64',
29 'windows-x86_64', 'windows-aarch64']
30# Windows 7 to 11 on x86_64 (nightly's tier-3 win7 target), and Windows 11 on Arm.
31WINDOWS = {'x86_64': 'x86_64-win7-windows-gnu', 'aarch64': 'aarch64-pc-windows-gnullvm'}
32# Clover's Developer ID Application certificate, by its SHA-1 hash: its name is the account
33# holder's legal name, which nothing here prints or stores.
34IDENTITY = 'BA308AA3591299E053E8824CEF1651F686F8908E'
35# The App Store Connect API key that notarizes it: {"key": P8 PATH, "key_id": ID, "issuer": ID}.
36NOTARY = Path('~/.config/snowbound/notary.json').expanduser()
37# The first published build's commit, where changes start when no build was published before.
38FIRST = '354f001dec3d731a4d1a6fac0a25d9e28550d781'
39KINDS = {'feat': 'feature', 'fix': 'fix'}
40
41
42def derive(release, commits):
43 """The version of the commit made at `release`, given when each commit leading to it was
44 made, itself included: its day in Los Angeles, and how many of those were made that day."""
45 day = release.astimezone(ZONE).date()
46 return day.isoformat(), sum(1 for made in commits if made.astimezone(ZONE).date() == day)
47
48
49def name(version):
50 return f'{version[0]}-r{version[1]}'
51
52
53def folder(version):
54 """The build's top-level folder."""
55 return f'{version[0]}.r{version[1]}'
56
57
58def parse(text):
59 date, revision = text.split('-r')
60 return date, int(revision)
61
62
63def newest(latest, archives, version):
64 """`latest.json`'s map with each of `archives`' platforms moved to `version` where that is
65 newer than what it names."""
66 return {**latest, **{platform: name(version) for platform in archives
67 if platform not in latest or parse(latest[platform]) < version}}
68
69
70def builds(published):
71 """The versions of the builds `published` holds, oldest first."""
72 return sorted(parse(entry.name.replace('.r', '-r')) for entry in published.iterdir()
73 if re.fullmatch(r'\d{4}-\d{2}-\d{2}\.r\d+', entry.name))
74
75
76def jj(*args):
77 return subprocess.check_output(['jj', *args], cwd=ROOT, text=True)
78
79
80def clean(dry_run, moment):
81 """Refuses to go on, or in a dry run warns, where the working copy isn't `main`."""
82 changed = jj('diff', '--from', 'main', '--to', '@', '--summary').strip()
83 if changed and not dry_run:
84 sys.exit(f'The working copy differs from main {moment}:\n{changed}')
85 if changed:
86 print(f'Dry run: the working copy differs from main {moment}.', file=sys.stderr)
87
88
89def run(command, **kwargs):
90 displayed = ['xcrun', 'notarytool', '[arguments redacted]'] if command[:2] == ['xcrun', 'notarytool'] else command
91 print('+', ' '.join(map(str, displayed)), flush=True)
92 try:
93 subprocess.run(command, cwd=ROOT, check=True, **kwargs)
94 except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:
95 if displayed is not command:
96 error.cmd = displayed
97 position = 1 if isinstance(error, subprocess.CalledProcessError) else 0
98 error.args = tuple(displayed if index == position else value
99 for index, value in enumerate(error.args))
100 raise
101
102
103def history():
104 """Every commit on `main` by id: its parents' ids, when it was made, and its message."""
105 template = ('commit_id ++ "\\x1f" ++ parents.map(|parent| parent.commit_id()).join(" ") ++ "\\x1f" ++ '
106 'committer.timestamp().utc().format("%Y-%m-%dT%H:%M:%S+00:00") ++ "\\x1f" ++ description ++ "\\x1e"')
107 commits = {}
108 for record in jj('log', '--no-graph', '-r', '::main', '-T', template).split('\x1e')[:-1]:
109 commit, parents, made, description = record.split('\x1f')
110 commits[commit] = (parents.split(), datetime.fromisoformat(made), description)
111 return commits
112
113
114def ancestors(commits, commit):
115 """`commit` and every commit before it."""
116 found, stack = set(), [commit]
117 while stack:
118 commit = stack.pop()
119 if commit not in found:
120 found.add(commit)
121 stack.extend(commits[commit][0])
122 return found
123
124
125def version_of(commits, commit):
126 return derive(commits[commit][1], [commits[each][1] for each in ancestors(commits, commit)])
127
128
129def entries(description):
130 """What a commit brings, as (kind, title): one entry of its prefix's kind, or one per item where
131 its body has a top-level bulleted list, each of the prefix's kind."""
132 subject, _, body = description.strip().partition('\n')
133 prefix = re.match(r'(\w+)(\([^)]*\))?!?:\s*', subject)
134 kind = KINDS.get(prefix[1].lower(), 'other') if prefix else 'other'
135 items, open_item = [], False
136 for line in body.splitlines():
137 if line.startswith(('- ', '* ')):
138 items.append(line[2:].strip())
139 open_item = True
140 elif open_item and line[:1].isspace() and line.strip():
141 items[-1] += ' ' + line.strip()
142 else:
143 open_item = False
144 titles = [title.rstrip('.') for title in items or [subject[prefix.end():] if prefix else subject]]
145 # Capitalized as a sentence, except a word like macOS or iCloud.
146 return [(kind, title if re.match(r'\S+[A-Z]', title) else title[:1].upper() + title[1:])
147 for title in titles if title]
148
149
150def changes(commits, commit, since):
151 """Every entry the commits after `since` up to `commit` bring, oldest first, each with the
152 version of the commit that brought it."""
153 versions = {each: version_of(commits, each)
154 for each in ancestors(commits, commit) - ancestors(commits, since)}
155 return [{'version': name(versions[each]), 'kind': kind, 'title': title}
156 for each in sorted(versions, key=versions.get) for kind, title in entries(commits[each][2])]
157
158
159def sign(files):
160 output = subprocess.check_output(
161 ['cargo', 'run', '--quiet', '--release', '-p', 'snowbound', '--example', 'release_sign', '--', KEY, *files],
162 cwd=ROOT, text=True)
163 return output.split()
164
165
166def minisign(files):
167 """Writes FILE.minisig beside each of `files` as `minisign -S` would with the release key:
168 a signature of the file's BLAKE2b-512, then one of that and the trusted comment."""
169 key_id = base64.b64decode(MINISIGN.read_text().splitlines()[1])[2:10]
170 comments = [f'timestamp:{int(time.time())}\tfile:{file.name}\thashed' for file in files]
171 with tempfile.TemporaryDirectory() as scratch:
172 def signed(messages):
173 paths = [Path(scratch) / str(index) for index in range(len(messages))]
174 for path, message in zip(paths, messages):
175 path.write_bytes(message)
176 return [bytes.fromhex(signature) for signature in sign(paths)]
177 signatures = signed([hashlib.blake2b(file.read_bytes()).digest() for file in files])
178 global_signatures = signed([signature + comment.encode() for signature, comment in zip(signatures, comments)])
179 for file, signature, comment, global_signature in zip(files, signatures, comments, global_signatures):
180 Path(f'{file}.minisig').write_text(
181 'untrusted comment: signature from the Snowbound release key\n'
182 f'{base64.b64encode(b"ED" + key_id + signature).decode()}\n'
183 f'trusted comment: {comment}\n{base64.b64encode(global_signature).decode()}\n')
184
185
186def split_debug(executable, debug):
187 """Moves `executable`'s debug info to `debug`, which its debug link then names."""
188 sysroot = subprocess.check_output(['rustc', '--print', 'sysroot'], text=True).strip()
189 host = re.search(r'^host: (\S+)$', subprocess.check_output(['rustc', '-vV'], text=True), re.M)[1]
190 objcopy = Path(sysroot) / 'lib/rustlib' / host / 'bin/rust-objcopy'
191 run([objcopy, '--only-keep-debug', executable, debug])
192 run([objcopy, '--strip-debug', f'--add-gnu-debuglink={debug}', executable])
193
194
195def zip_bundle(bundle, archive):
196 run(['ditto', '-c', '-k', '--norsrc', '--noextattr', '--noqtn', '--noacl', '--keepParent', bundle, archive])
197
198
199def notary():
200 """notarytool's credential arguments from NOTARY, if they sign in."""
201 if not NOTARY.exists():
202 return None
203 key = json.loads(NOTARY.read_text())
204 arguments = ['--key', str(Path(key['key']).expanduser()), '--key-id', key['key_id'], '--issuer', key['issuer']]
205 signs_in = subprocess.run(['xcrun', 'notarytool', 'history', *arguments], capture_output=True).returncode == 0
206 return arguments if signs_in else None
207
208
209def staple(bundle):
210 for attempt in range(5):
211 with tempfile.TemporaryFile(mode='w+') as log:
212 try:
213 try:
214 run(['xcrun', 'stapler', 'staple', bundle], stdout=log,
215 stderr=subprocess.STDOUT, timeout=120)
216 finally:
217 log.seek(0)
218 diagnostic = log.read()
219 print(diagnostic, end='', flush=True)
220 except subprocess.CalledProcessError as error:
221 # EX_NOHOST alone also means the bundle has never been notarized.
222 if attempt == 4 or error.returncode != 68 or not re.search(
223 r'^error is Error Domain=NSURLErrorDomain Code=-1005(?:\s|$)', diagnostic, re.M):
224 raise
225 time.sleep(10 * 2 ** attempt)
226 else:
227 return
228
229
230def build_mac(platform, folder, developer_id, notarize, symbols):
231 """The zipped app, which build_macos.py signs; 10.6's stays unsigned, as it predates Developer ID.
232 Its zipped dSYM goes to `symbols`."""
233 bundle = folder / 'Snowbound.app'
234 if platform == 'macos-10.6':
235 signing = ['--snow-leopard']
236 elif developer_id:
237 signing = ['--sign', 'developer-id', '--sign-identity', IDENTITY]
238 else:
239 signing = ['--sign', 'ad-hoc']
240 if platform != 'macos-10.6':
241 signing += ['--arch', platform.removeprefix('macos-')]
242 dsym = folder / 'Snowbound.dSYM'
243 run([sys.executable, ROOT / 'tools/canvas/build_macos.py', '--release', '--output', bundle, '--dsym', dsym, *signing])
244 zip_bundle(dsym, symbols)
245 archive = folder / 'archive.zip'
246 if notarize and platform != 'macos-10.6':
247 zip_bundle(bundle, archive)
248 run(['xcrun', 'notarytool', 'submit', archive, *notarize, '--wait'])
249 staple(bundle)
250 archive.unlink()
251 zip_bundle(bundle, archive)
252 return archive
253
254
255def build_linux(architectures):
256 """The executables, each all of Snowbound for its architecture."""
257 run(['sh', ROOT / 'crates/snowbound/linux/package.sh', *architectures])
258 return {f'linux-{arch}': ROOT / f'target/{arch}-unknown-linux-gnu/release/snowbound' for arch in architectures}
259
260
261def build_windows(architectures):
262 """The executables, each all of Snowbound for its architecture, one running on every
263 Windows it supports."""
264 built = {}
265 for arch in architectures:
266 clean_local(ROOT, '--release', command=['sh', ROOT / 'platform/windows/cargo.sh', arch])
267 run(['sh', ROOT / 'platform/windows/cargo.sh', arch, 'build', '--release', '-p', 'snowbound'])
268 built[f'windows-{arch}'] = ROOT / f'target/windows/{WINDOWS[arch]}/release/snowbound.exe'
269 return built
270
271
272def copy_download(source, destination):
273 shutil.copyfile(source, destination)
274 try:
275 shutil.copymode(source, destination)
276 except PermissionError as error:
277 print(f'Warning: copied {destination}, but could not preserve file permissions: {error}',
278 file=sys.stderr)
279
280
281def main():
282 parser = argparse.ArgumentParser(description=__doc__)
283 parser.add_argument('--dry-run', action='store_true',
284 help='Publish into a new temporary folder instead, even from a changed working copy')
285 parser.add_argument('--platforms', nargs='+', choices=PLATFORMS, default=PLATFORMS)
286 parser.add_argument('--ad-hoc', action='store_true',
287 help='Sign the macOS app ad hoc instead of with Developer ID, unnotarized')
288 args = parser.parse_args()
289 developer_id = not args.ad_hoc and any(platform in ('macos-aarch64', 'macos-x86_64') for platform in args.platforms)
290 identities = subprocess.run(['security', 'find-identity', '-v', '-p', 'codesigning'],
291 capture_output=True, text=True).stdout
292 if developer_id and IDENTITY not in identities:
293 sys.exit(f'The keychain has no signing identity {IDENTITY}; release with --ad-hoc, or add it.')
294 notarize = notary() if developer_id else None
295 if developer_id and not notarize:
296 print(f'Not notarizing: no API key in {NOTARY} that signs in (see tools/RELEASE.md).',
297 file=sys.stderr)
298
299 commit = jj('log', '--no-graph', '-r', 'main', '-T', 'commit_id').strip()
300 clean(args.dry_run, 'to release it')
301 commits = history()
302 version = version_of(commits, commit)
303 print(f'Snowbound build {version[0]} revision {version[1]}, commit {commit}', flush=True)
304
305 published = Path(tempfile.mkdtemp(prefix='snowbound-release-')) if args.dry_run else PUBLISHED
306 if not published.is_dir():
307 sys.exit(f'{published} is not mounted.')
308 target = published / folder(version)
309 if target.exists():
310 build = json.loads((target / 'build.json').read_text())
311 if build['commit'] != commit:
312 sys.exit(f'{target} holds commit {build["commit"]}, not {commit}.')
313 print(f'{target} is already published.')
314 else:
315 run([sys.executable, ROOT / 'tools/ci.py', '--rev', commit])
316 clean(args.dry_run, 'after the checks')
317 stage = ROOT / 'target/release-stage' / name(version)
318 shutil.rmtree(stage, ignore_errors=True)
319 stage.mkdir(parents=True)
320 # The app reads its version from this as it compiles.
321 os.environ['SNOWBOUND_BUILD'] = name(version)
322 # For the symbol files; the executables shed it.
323 os.environ['CARGO_PROFILE_RELEASE_DEBUG'] = 'line-tables-only'
324 built = {}
325 symbols = []
326 for platform in args.platforms:
327 if platform.startswith('macos'):
328 work = stage / platform
329 work.mkdir()
330 symbols.append(stage / f'Snowbound-{name(version)}-{platform}.dSYM.zip')
331 built[platform] = build_mac(platform, work, developer_id, notarize, symbols[-1])
332 linux = [platform.removeprefix('linux-') for platform in args.platforms if platform.startswith('linux')]
333 if linux:
334 built |= build_linux(linux)
335 windows = [platform.removeprefix('windows-') for platform in args.platforms if platform.startswith('windows')]
336 if windows:
337 built |= build_windows(windows)
338 clean(args.dry_run, 'after the build')
339 files = {}
340 for platform, source in built.items():
341 prefix = 'Snowbound' if platform.startswith('macos') else 'snowbound'
342 files[platform] = stage / f'{prefix}-{name(version)}-{platform}{source.suffix}'
343 shutil.copy2(source, files[platform])
344 if not platform.startswith('macos'):
345 debug = stage / f'{prefix}-{name(version)}-{platform}.debug'
346 split_debug(files[platform], debug)
347 symbols.append(debug.with_name(f'{debug.name}.zip'))
348 with zipfile.ZipFile(symbols[-1], 'w', zipfile.ZIP_DEFLATED) as archive:
349 archive.write(debug, debug.name)
350 clean_local(ROOT, '--release')
351 signatures = sign(files.values())
352 # A dry run's changes too start after the newest build the share holds.
353 before = [each for each in builds(PUBLISHED) if each < version] if PUBLISHED.is_dir() else []
354 since = json.loads((PUBLISHED / folder(before[-1]) / 'build.json').read_text())['commit'] if before else FIRST
355 build = {
356 'version': name(version),
357 'commit': commit,
358 'published': datetime.now(ZONE).isoformat(timespec='seconds'),
359 'changes': changes(commits, commit, since),
360 'archives': {platform: {
361 'file': file.name,
362 'size': file.stat().st_size,
363 'sha256': hashlib.sha256(file.read_bytes()).hexdigest(),
364 # Older apps check it; newer ones trust the sha256 build.json.sig vouches for.
365 'signature': signature,
366 } for (platform, file), signature in zip(files.items(), signatures)},
367 }
368 (stage / 'build.json').write_text(json.dumps(build, indent=2) + '\n')
369 (stage / 'build.json.sig').write_text(sign([stage / 'build.json'])[0] + '\n')
370 downloads = [*files.values(), *symbols, stage / 'build.json']
371 minisign(downloads)
372 partial = target.with_name(f'.{target.name}.partial')
373 shutil.rmtree(partial, ignore_errors=True)
374 partial.mkdir()
375 for file in [*downloads, *(Path(f'{file}.minisig') for file in downloads), stage / 'build.json.sig']:
376 copy_download(file, partial / file.name)
377 partial.rename(target)
378 shutil.rmtree(stage)
379 print(f'Published {target}')
380
381 history_file = published / 'history.json'
382 partial = history_file.with_name('.history.json.partial')
383 partial.write_text(json.dumps([name(each) for each in builds(published)], indent=2) + '\n')
384 os.replace(partial, history_file)
385 latest_file = published / 'latest.json'
386 latest = json.loads(latest_file.read_text()) if latest_file.exists() else {}
387 build = json.loads((target / 'build.json').read_text())
388 moved = newest(latest, build['archives'], version)
389 if moved != latest:
390 partial = latest_file.with_name('.latest.json.partial')
391 partial.write_text(json.dumps(moved, indent=2, sort_keys=True) + '\n')
392 os.replace(partial, latest_file)
393 print(f'{latest_file}: {json.dumps(moved, sort_keys=True)}')
394 # Stable names for the readme's download links, always the newest build of each platform.
395 downloads = published / 'latest'
396 downloads.mkdir(exist_ok=True)
397 for platform, newest_name in moved.items():
398 if newest_name != name(version):
399 continue
400 file = build['archives'][platform]['file']
401 for published_name in (file, f'{file}.minisig'):
402 stable = downloads / published_name.replace(f'-{name(version)}', '')
403 partial = stable.with_name(f'.{stable.name}.partial')
404 copy_download(target / published_name, partial)
405 os.replace(partial, stable)
406 if not args.dry_run:
407 print(f'{URL}{folder(version)}/')
408
409
410if __name__ == '__main__':
411 main()