1#!/usr/bin/env python3
2"""Gates a revision (main by default) on formatting, Clippy, the tests, the Python suite and
3every platform's build, checked out in the jj workspace workspaces/ci so that edits in
4progress elsewhere never reach it. See tools/TESTING.md."""
5import argparse
6from concurrent.futures import ThreadPoolExecutor
7from datetime import datetime
8import fcntl
9import json
10import os
11from pathlib import Path
12import re
13import shutil
14import signal
15import subprocess
16import sys
17import threading
18import time
19
20from cargo_fresh import local_packages
21
22ROOT = Path(__file__).resolve().parents[1]
23# The checkout holding the jj repository, which a nested workspace's `.jj/repo` file names.
24STORE = ROOT / '.jj/repo'
25MAIN = (STORE.parent / STORE.read_text()).resolve().parents[1] if STORE.is_file() else ROOT
26CI = MAIN / 'workspaces/ci'
27TARGET = CI / 'target'
28RUNS = TARGET / 'ci'
29# Changes here rebuild every package.
30EVERYTHING = ('Cargo.toml', 'Cargo.lock', 'vendor/', 'rust-toolchain', '.cargo/')
31JSON = '--message-format=json'
32
33
34def cargo(*args):
35 return ['cargo', args[0], '--locked', *([] if args[0] == 'clean' else [JSON]), *args[1:]]
36
37
38def cross(script, arch, *args):
39 return ['sh', script, arch, *cargo(*args)[1:]]
40
41
42def lanes():
43 """Each lane: its commands, the packages whose dependencies (None for all) and the paths
44 that select it under --changed, extra environment, and why it can't run here, if so."""
45 clippy = ['clippy', '--workspace', '--all-targets', '--all-features', '--', '-D', 'warnings']
46 # What ships: mobile is the phone apps', with lanes of their own, and the tests and lab
47 # examples run only on macOS.
48 cross_clippy = ['clippy', '--workspace', '--exclude', 'mobile', '--lib', '--bins', '--all-features',
49 '--', '-D', 'warnings']
50 windows = CI / 'platform/windows/cargo.sh'
51 linux = CI / 'platform/linux/cargo.sh'
52 mingw = os.environ.get('LLVM_MINGW') or MAIN / 'target/windows/llvm-mingw'
53 sdk = Path(os.environ.get('SNOW_LEOPARD_SDK') or MAIN / 'target/snow-leopard/MacOSX10.6.sdk')
54 nightly = subprocess.run(['rustup', 'component', 'list', '--installed', '--toolchain', 'nightly'],
55 capture_output=True, text=True).stdout.split()
56 result = [
57 dict(name='test', minutes=45, packages=None, paths=()),
58 dict(name='clippy', minutes=30, packages=None, paths=(), commands=[
59 cargo(*clippy),
60 cargo('clippy', '-p', 'snowbound', '--no-default-features', '--', '-D', 'warnings')]),
61 dict(name='fmt', minutes=5, packages=None, paths=(), commands=[['cargo', 'fmt', '--all', '--check']]),
62 dict(name='python', minutes=30, packages=None, paths=('tools/', 'corpus/'), commands=[
63 # The suite runs this checkout's examples from target/debug.
64 cargo('build', '--workspace', '--all-features', '--examples', '--bins'),
65 ['uv', 'run', '--no-project', '--python', '3.12', '--with', 'pillow', '--with', 'pdfplumber',
66 'python', '-m', 'unittest', 'discover', '-s', 'tools', '-p', 'test_*.py']],
67 environment={'PYTHONPATH': str(CI / 'tools')},
68 missing=None if shutil.which('uv') else 'needs uv'),
69 ]
70 for arch in ('x86_64', 'aarch64'):
71 result.append(dict(
72 name=f'windows-{arch}', minutes=45, packages=['snowbound'], paths=('platform/windows/',),
73 clean=cross(windows, arch, 'clean'),
74 # x86_64's nightly Clippy would lint std, which it builds, and nightly's own new lints.
75 commands=[*([] if arch == 'x86_64' else [cross(windows, arch, *cross_clippy)]),
76 cross(windows, arch, 'build', '-p', 'snowbound')],
77 environment={'LLVM_MINGW': str(mingw)},
78 missing=None if arch == 'aarch64' or 'rust-src' in nightly else 'needs nightly with rust-src'))
79 for arch in ('x86_64', 'aarch64'):
80 result.append(dict(
81 name=f'linux-{arch}', minutes=30, packages=['snowbound'], paths=('platform/linux/',),
82 clean=cross(linux, arch, 'clean'),
83 commands=[cross(linux, arch, *cross_clippy), cross(linux, arch, 'build', '-p', 'snowbound')],
84 environment={'CARGO_TARGET_DIR': str(TARGET / 'linux')},
85 missing=None if shutil.which('zig') else 'needs zig'))
86 result.append(dict(
87 name='web-js', minutes=5, packages=['notebook'], paths=('tools/web/',),
88 commands=[['node', '--test', 'tools/web/test_*.mjs']],
89 missing=None if shutil.which('node') else 'needs node'))
90 targets = subprocess.run(['rustup', 'target', 'list', '--installed'], capture_output=True, text=True).stdout
91 wasm = web_environment()
92 result.append(dict(
93 name='web', minutes=20, packages=['snowbound'], paths=('crates/snowbound/web/', 'tools/release_web.py', 'tools/web/'),
94 clean=cargo('clean', '--target', 'wasm32-unknown-unknown'),
95 # Clippy builds it; the module itself is linked by release_web.py.
96 commands=[cargo('clippy', '-p', 'snowbound', '--target', 'wasm32-unknown-unknown', '--no-default-features', '--features', 'wgpu,live', '--', '-D', 'warnings'),
97 ['python3', 'tools/web/receipt_probe.py']],
98 environment={**(wasm or {}), 'CARGO_TARGET_DIR': str(TARGET / 'wasm')},
99 missing='needs the wasm32-unknown-unknown target' if 'wasm32-unknown-unknown' not in targets.split()
100 else None if wasm else 'needs nix for a clang that builds for wasm32'))
101 result.append(dict(
102 name='ios', minutes=30, packages=['mobile'], paths=('apps/ios/',),
103 # build-rust.sh builds the Rust half into target/ios.
104 commands=[['xcodebuild', '-quiet', '-project', 'apps/ios/Snowbound.xcodeproj', '-scheme', 'Snowbound',
105 '-configuration', 'Debug', '-destination', 'generic/platform=iOS Simulator',
106 '-derivedDataPath', TARGET / 'ios-xcode', 'CODE_SIGNING_ALLOWED=NO', 'build']],
107 missing=None if shutil.which('xcodebuild') else 'needs Xcode'))
108 ndk, jdk, sdks, rustup, missing = android_toolchain()
109 result.append(dict(
110 name='android', minutes=45, packages=['mobile'], paths=('apps/android/', 'platform/android/'),
111 # build.py cleans its own artifacts, as it reads the API level from the app's manifest.
112 commands=[['python3', 'apps/android/build.py', '--clippy', *(arg for sdk in sdks for arg in ('--sdk', sdk)),
113 '--ndk', ndk, '--java-home', jdk, '--output', TARGET / 'android-apk']],
114 environment={'ANDROID_NDK_HOME': str(ndk), **({'RUSTUP_HOME': str(rustup)} if rustup else {})},
115 missing=missing))
116 result.append(dict(
117 name='macos-10.6', minutes=45, packages=['snowbound'], paths=('platform/snow-leopard/',),
118 clean=['sh', CI / 'platform/snow-leopard/cargo.sh', 'clean', '--locked'],
119 commands=[['sh', CI / 'platform/snow-leopard/cargo.sh', 'build', '--locked', JSON, '-p', 'snowbound',
120 '--no-default-features']],
121 environment={'SNOW_LEOPARD_SDK': str(sdk)},
122 missing=('needs nightly with rust-src' if 'rust-src' not in nightly else
123 None if sdk.exists() else 'needs the 10.6 SDK from platform/snow-leopard/remote.sh sdk')))
124 return result
125
126
127def android_toolchain():
128 """The NDK, JDK and SDK folders the Android lane builds with, as ANDROID_NDK_HOME,
129 JAVA_HOME and ANDROID_HOME name them or target/android-toolchain holds them; a rustup home
130 for the Android target where the default lacks it; and why the lane can't run, if so."""
131 tools = MAIN / 'target/android-toolchain'
132
133 def found(variable, *patterns):
134 named = os.environ.get(variable)
135 candidates = [Path(named)] if named else [path for pattern in patterns for path in sorted(tools.glob(pattern))]
136 return next((path for path in candidates if path.is_dir()), None)
137
138 ndk = found('ANDROID_NDK_HOME', 'android-ndk-*')
139 jdk = found('JAVA_HOME', 'jdk-*/Contents/Home', 'jdk-*')
140 sdks = [path for path in (os.environ.get('ANDROID_HOME'), Path.home() / 'Library/Android/sdk',
141 tools / 'provider-sdk') if path and Path(path).is_dir()]
142 target = 'aarch64-linux-android'
143 installed = subprocess.run(['rustup', 'target', 'list', '--installed'], capture_output=True, text=True).stdout
144 rustup = None if target in installed.split() else tools / 'rustup'
145 missing = (f'needs the {target} target' if rustup and not any(rustup.glob(f'toolchains/*/lib/rustlib/{target}'))
146 else 'needs the Android NDK' if not ndk else 'needs a JDK' if not jdk
147 else 'needs the Android SDK' if not sdks else None)
148 return ndk, jdk, sdks, rustup, missing
149
150
151def web_environment():
152 """A clang and llvm-ar that build SQLite for wasm32, as release_web.py finds them; none
153 where it finds none."""
154 import release_web
155 try:
156 environment = release_web.environment()
157 except SystemExit:
158 return None
159 return {key: environment[key] for key in ('CC_wasm32_unknown_unknown', 'AR_wasm32_unknown_unknown')}
160
161
162def jj(*args, cwd=ROOT):
163 return subprocess.run(['jj', *args], cwd=cwd, check=True, capture_output=True, text=True).stdout
164
165
166def checkout(rev):
167 """Points the CI workspace's own commit, a child of main, at `rev`'s files."""
168 commits = jj('log', '--no-graph', '-r', rev, '-T', 'commit_id ++ "\\n"').split()
169 if len(commits) != 1:
170 sys.exit(f'{rev} names {len(commits)} revisions, not one.')
171 if not CI.exists():
172 CI.parent.mkdir(exist_ok=True)
173 jj('workspace', 'add', '--name', 'ci', '-r', 'main', str(CI))
174 jj('workspace', 'update-stale', cwd=CI)
175 jj('rebase', '-r', '@', '-o', 'main', cwd=CI)
176 jj('restore', '--from', commits[0], cwd=CI)
177 return commits[0]
178
179
180class Run:
181 def __init__(self, folder, environment):
182 self.folder = folder
183 self.environment = environment
184 self.processes = set()
185 self.lock = threading.Lock()
186 self.stopping = False
187
188 def execute(self, command, log, deadline, environment=(), cwd=CI):
189 """The command's exit status, or None where it outlived `deadline`."""
190 with open(log, 'a') as stream:
191 stream.write(f'$ {" ".join(map(str, command))}\n')
192 stream.flush()
193 try:
194 process = subprocess.Popen(list(map(str, command)), cwd=cwd, stdout=stream,
195 stderr=subprocess.STDOUT, start_new_session=True,
196 env={**self.environment, **dict(environment)})
197 except OSError as error:
198 stream.write(f'error: {error}\n')
199 return 127
200 with self.lock:
201 self.processes.add(process)
202 if self.stopping:
203 stop(process)
204 try:
205 return process.wait(timeout=max(deadline - time.monotonic(), 0))
206 except subprocess.TimeoutExpired:
207 stop(process)
208 stream.write('\n(timed out)\n')
209 return None
210 finally:
211 with self.lock:
212 self.processes.discard(process)
213
214 def stop_all(self):
215 with self.lock:
216 self.stopping = True
217 for process in self.processes:
218 stop(process)
219
220
221def stop(process):
222 for sent in (signal.SIGTERM, signal.SIGKILL):
223 try:
224 os.killpg(process.pid, sent)
225 process.wait(timeout=5)
226 return
227 except (ProcessLookupError, subprocess.TimeoutExpired):
228 pass
229
230
231def status(code):
232 return 'passed' if code == 0 else 'timeout' if code is None else 'failed'
233
234
235def run_lane(run, lane, deadline):
236 log = run.folder / f'{lane["name"]}.log'
237 for command in lane['commands']:
238 code = run.execute(command, log, deadline, lane.get('environment', {}))
239 if code != 0:
240 return status(code), [log]
241 return 'passed', [log]
242
243
244def run_tests(run, deadline, packages, jobs):
245 """Builds every test target, then runs those of `packages` (all where None) side by side,
246 each from its package's folder as cargo would, beside the doctests."""
247 log = run.folder / 'test.log'
248 code = run.execute(cargo('test', '--workspace', '--all-features', '--no-run'), log, deadline)
249 if code != 0:
250 return status(code), [log], []
251 binaries = []
252 for line in log.read_text(errors='replace').splitlines():
253 message = json.loads(line) if line.startswith('{') else {}
254 if message.get('reason') == 'compiler-artifact' and message['profile']['test'] and message['executable']:
255 folder = Path(message['manifest_path']).parent
256 if packages is None or folder.name in packages:
257 binaries.append((f'{folder.name}.{message["target"]["name"]}.'
258 f'{"-".join(message["target"]["kind"])}', [message['executable']], folder))
259 binaries.append(('doctests', ['cargo', 'test', '--locked', '--workspace', '--all-features', '--doc'], CI))
260 # The slowest last time start first, so that none of them starts last.
261 timings = RUNS / 'test-seconds.json'
262 seconds = json.loads(timings.read_text()) if timings.exists() else {}
263 binaries.sort(key=lambda binary: -seconds.get(binary[0], float('inf')))
264 (run.folder / 'test').mkdir()
265
266 def one(binary):
267 name, command, cwd = binary
268 started = time.monotonic()
269 part_log = run.folder / 'test' / f'{name}.log'
270 code = run.execute(command, part_log, deadline, cwd=cwd)
271 return {'name': name, 'status': status(code), 'seconds': round(time.monotonic() - started, 1),
272 'log': str(part_log)}
273
274 with ThreadPoolExecutor(jobs) as pool:
275 parts = list(pool.map(one, binaries))
276 timings.write_text(json.dumps({**seconds, **{part['name']: part['seconds'] for part in parts}}, indent=1))
277 failed = [part for part in parts if part['status'] != 'passed']
278 result = 'timeout' if any(part['status'] == 'timeout' for part in failed) else 'failed' if failed else 'passed'
279 return result, [log, *(Path(part['log']) for part in failed)], sorted(parts, key=lambda part: -part['seconds'])
280
281
282def relative(path):
283 return path.removeprefix(f'{CI}/')
284
285
286def diagnose(log):
287 """The errors a log reports, each with the file and line it names where it names one."""
288 errors, text = [], []
289 lines = log.read_text(errors='replace').splitlines()
290 test = None
291 for index, line in enumerate(lines):
292 following = lines[index + 1] if index + 1 < len(lines) else ''
293 if line.startswith('{'):
294 try:
295 message = json.loads(line).get('message') or {}
296 except json.JSONDecodeError:
297 continue
298 spans = [span for span in message.get('spans', []) if span.get('is_primary')]
299 if message.get('level') == 'error' and spans:
300 errors.append({'error': message['message'],
301 'file': f'{relative(spans[0]["file_name"])}:{spans[0]["line_start"]}'})
302 elif match := re.match(r'---- (\S+) stdout ----', line):
303 test = match[1]
304 elif match := re.search(r'panicked at (\S+?):(\d+):\d+:$', line):
305 errors.append({'error': f'{test or "test"} panicked: {following.strip()}',
306 'file': f'{relative(match[1])}:{match[2]}'})
307 elif match := re.match(r'Diff in (\S+?):(\d+):', line):
308 errors.append({'error': 'not formatted (cargo fmt)', 'file': f'{relative(match[1])}:{match[2]}'})
309 elif match := re.match(r'(FAIL|ERROR): (\S+) \((\S+)\)', line):
310 block = lines[index + 2:]
311 end = next((i for i, rest in enumerate(block) if rest.startswith(('=' * 20, '-' * 20))), len(block))
312 block = [rest for rest in block[:end] if rest.strip()]
313 frames = [frame for frame in block if f'File "{CI}/' in frame]
314 where = re.search(r'File "(\S+)", line (\d+)', frames[-1]) if frames else None
315 errors.append({'error': f'{match[3]}: {block[-1].strip() if block else match[1]}',
316 'file': f'{relative(where[1])}:{where[2]}' if where else None})
317 elif match := re.match(r'(/\S+?):(\d+):\d+: (?:fatal )?error: (.*)', line):
318 errors.append({'error': match[3], 'file': f'{relative(match[1])}:{match[2]}'})
319 elif (match := re.match(r'error(?:\[\w+\])?: (.*)', line)) and (
320 where := re.match(r'\s*--> (\S+?):(\d+):\d+', following)):
321 errors.append({'error': match[1], 'file': f'{relative(where[1])}:{where[2]}'})
322 elif re.match(r'(error|test \S+ \.\.\. FAILED)', line):
323 text.append({'error': line.strip(), 'file': None})
324 if not errors and not text:
325 text = [{'error': line, 'file': None} for line in lines if line.strip() and not line.startswith('{')][-3:]
326 return list({(error['error'], error['file']): error for error in errors or text}.values())
327
328
329def affected(changed, metadata):
330 """The workspace packages that changed files reach, through what depends on them."""
331 folders = {package['name']: Path(package['manifest_path']).parent.relative_to(CI).as_posix()
332 for package in metadata['packages']}
333 if any(path.startswith(EVERYTHING) for path in changed):
334 return set(folders)
335 reached = {name for name, folder in folders.items() if any(path.startswith(folder + '/') for path in changed)}
336 users = {package['name']: {dependency['name'] for dependency in package['dependencies']
337 if dependency['name'] in folders} for package in metadata['packages']}
338 while more := {name for name, uses in users.items() if uses & reached} - reached:
339 reached |= more
340 return reached
341
342
343def dependencies(roots, metadata):
344 uses = {package['name']: {dependency['name'] for dependency in package['dependencies']}
345 for package in metadata['packages']}
346 closure, pending = set(), list(roots)
347 while pending:
348 name = pending.pop()
349 if name in uses and name not in closure:
350 closure.add(name)
351 pending.extend(uses[name])
352 return closure
353
354
355def size(path):
356 if not path.is_dir() or path.is_symlink():
357 return path.lstat().st_blocks * 512
358 return sum(file.lstat().st_blocks * 512 for file in path.rglob('*'))
359
360
361def prune(budget, started):
362 """Deletes, least recently used first, the build units this run didn't use until the
363 target folder fits `budget` bytes. Cargo reads a unit's fingerprint whenever it checks
364 it, so the fingerprint's access time is when a build last used the unit."""
365 total = disk_usage()
366 if total <= budget:
367 return 0
368 units = {}
369 for fingerprints in [*TARGET.glob('*/.fingerprint'), *TARGET.glob('*/*/.fingerprint'),
370 *TARGET.glob('*/*/*/.fingerprint')]:
371 profile = fingerprints.parent
372 for kind in ('deps', 'build', '.fingerprint', 'incremental', 'examples'):
373 for entry in os.scandir(profile / kind) if (profile / kind).is_dir() else ():
374 if match := re.search(r'-([0-9a-f]{16})(?:\.|$)', entry.name):
375 units.setdefault((profile, match[1]), []).append(Path(entry.path))
376 used = {}
377 for unit, paths in units.items():
378 stamps = [file.stat().st_atime for path in paths if path.parent.name == '.fingerprint'
379 for file in path.iterdir()] or [path.stat().st_atime for path in paths]
380 used[unit] = max(stamps)
381 freed = 0
382 for unit in sorted(units, key=used.get):
383 if used[unit] >= started or total - freed <= budget * 0.8:
384 break
385 for path in units[unit]:
386 freed += size(path)
387 shutil.rmtree(path) if path.is_dir() and not path.is_symlink() else path.unlink()
388 return total - disk_usage()
389
390
391def disk_usage():
392 return int(subprocess.check_output(['du', '-sk', TARGET]).split()[0]) * 1024
393
394
395def table(results):
396 lines = []
397 for result in results:
398 seconds = result.get('seconds')
399 clock = f'{int(seconds // 60)}:{int(seconds % 60):02}' if seconds is not None else ''
400 note = result.get('note', '')
401 lines.append(f'{result["name"]:<16} {result["status"]:<8} {clock:>6} {note}'.rstrip())
402 for error in result.get('errors', [])[:6]:
403 where = f'{error["file"]} ' if error['file'] else ''
404 lines.append(f'{"":<33}{where}{error["error"].splitlines()[0][:160]}')
405 if len(result.get('errors', [])) > 6:
406 lines.append(f'{"":<33}… {len(result["errors"]) - 6} more in {result["log"]}')
407 elif result['status'] in ('failed', 'timeout'):
408 lines.append(f'{"":<33}{result["log"]}')
409 return '\n'.join(lines)
410
411
412def main():
413 every = lanes()
414 names = [lane['name'] for lane in every]
415 parser = argparse.ArgumentParser(description=__doc__)
416 source = parser.add_mutually_exclusive_group()
417 source.add_argument('--rev', default='main', help='The jj revision to gate; main by default')
418 source.add_argument('--working-copy', action='store_true',
419 help="This checkout's working copy, as it is when the run starts")
420 parser.add_argument('--lanes', nargs='+', metavar='LANE',
421 help=f'Only these lanes, or those starting LANE-: {", ".join(names)}')
422 parser.add_argument('--changed', action='store_true',
423 help="Only the lanes, and tests of the packages, that the revision's changes from main reach")
424 parser.add_argument('--jobs', type=int, default=4, help='Lanes at once (default 4)')
425 parser.add_argument('--test-jobs', type=int, default=4, help='Test executables at once (default 4)')
426 parser.add_argument('--timeout', type=float, metavar='MINUTES', help="Each lane's limit, overriding its own")
427 parser.add_argument('--budget', type=float, default=40, metavar='GB',
428 help='Prune workspaces/ci/target to this size after the run (default 40)')
429 args = parser.parse_args()
430 if ROOT == CI:
431 sys.exit(f'Run ci.py from another checkout; {CI} is its own.')
432 def named(lane, name):
433 return lane == name or lane.startswith(f'{name}-')
434 unknown = [name for name in args.lanes or () if not any(named(lane, name) for lane in names)]
435 if unknown:
436 parser.error(f'unknown lanes {unknown}; choose from {names}')
437 chosen = [lane for lane in every if not args.lanes or any(named(lane['name'], name) for name in args.lanes)]
438
439 RUNS.mkdir(parents=True, exist_ok=True)
440 lock = open(RUNS / 'lock', 'w')
441 try:
442 fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
443 except BlockingIOError:
444 print('Waiting for another ci.py run to finish…', file=sys.stderr, flush=True)
445 fcntl.flock(lock, fcntl.LOCK_EX)
446 started_at, started = datetime.now().astimezone(), time.monotonic()
447 wall = time.time()
448 rev = '@' if args.working_copy else args.rev
449 commit = checkout(rev)
450 described = jj('log', '--no-graph', '-r', commit, '-T',
451 'change_id.short() ++ " " ++ commit_id.short() ++ " " ++ description.first_line()').strip()
452 print(f'Gating {rev}: {described}', flush=True)
453
454 tested = None
455 changed = None
456 if args.changed:
457 changed = [line for line in jj('diff', '--from', 'main', '--to', commit, '--name-only').splitlines() if line]
458 metadata = json.loads(subprocess.check_output(['cargo', 'metadata', '--format-version=1', '--no-deps'], cwd=CI))
459 tested = affected(changed, metadata)
460 chosen = [lane for lane in chosen
461 if any(path.startswith(lane['paths']) for path in changed)
462 or tested & (dependencies(lane['packages'], metadata) if lane['packages'] else tested)]
463 print(f'Changed from main: {len(changed)} files, reaching {", ".join(sorted(tested)) or "no packages"}',
464 flush=True)
465
466 folder = RUNS / 'runs' / started_at.strftime('%Y%m%d-%H%M%S')
467 folder.mkdir(parents=True)
468 environment = {key: value for key, value in os.environ.items()
469 if not key.startswith(('ONESTORE_', 'SNOWBOUND_', 'CARGO_TARGET_DIR'))}
470 run = Run(folder, environment)
471
472 ready = [lane for lane in chosen if not lane.get('missing')]
473 cleaning = ([(cargo('clean'), {})]
474 if any(lane['name'] in ('test', 'clippy', 'python') for lane in ready) else [])
475 cleaning += [(lane['clean'], lane.get('environment', {})) for lane in ready if 'clean' in lane]
476 packages = local_packages(CI, environment) if cleaning else []
477 for command, extra in cleaning:
478 command += [arg for package in packages for arg in ('-p', package)]
479 code = run.execute(command, folder / 'clean.log', time.monotonic() + 300, extra)
480 if code != 0:
481 lock.close()
482 sys.exit(f'Workspace cleanup {status(code)}: {folder / "clean.log"}')
483
484 def gate(lane):
485 result = {'name': lane['name']}
486 if lane.get('missing'):
487 print(f'{lane["name"]}: skipped, {lane["missing"]}', flush=True)
488 return {**result, 'status': 'skipped', 'note': lane['missing']}
489 began = time.monotonic()
490 deadline = began + 60 * (args.timeout or lane['minutes'])
491 if lane['name'] == 'test':
492 outcome, logs, parts = run_tests(run, deadline, tested, args.test_jobs)
493 result['parts'] = parts
494 else:
495 outcome, logs = run_lane(run, lane, deadline)
496 errors = [error for log in logs for error in diagnose(log)] if outcome != 'passed' else []
497 if outcome == 'timeout':
498 errors.insert(0, {'error': f'timed out after {args.timeout or lane["minutes"]:g} minutes', 'file': None})
499 seconds = round(time.monotonic() - began, 1)
500 print(f'{lane["name"]}: {outcome} in {seconds:.0f}s', flush=True)
501 return {**result, 'status': outcome, 'seconds': seconds, 'log': str(logs[0]), 'errors': errors}
502
503 pool = ThreadPoolExecutor(args.jobs)
504 try:
505 results = list(pool.map(gate, chosen))
506 except KeyboardInterrupt:
507 pool.shutdown(wait=False, cancel_futures=True)
508 run.stop_all()
509 raise
510 passed = all(result['status'] in ('passed', 'skipped') for result in results)
511 freed = prune(args.budget * 1e9, wall)
512 summary = {
513 'status': 'passed' if passed else 'failed', 'revision': rev, 'commit': commit, 'described': described,
514 'workspace': str(CI), 'started': started_at.isoformat(timespec='seconds'),
515 'seconds': round(time.monotonic() - started, 1), 'changed': changed,
516 'tested_packages': sorted(tested) if tested is not None else None,
517 'pruned_bytes': freed, 'lanes': results,
518 }
519 (folder / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n')
520 for old in sorted((RUNS / 'runs').iterdir())[:-20]:
521 shutil.rmtree(old)
522 print()
523 print(table(results))
524 print(f'\n{summary["status"]} in {summary["seconds"] / 60:.1f} min: {folder / "summary.json"}')
525 sys.exit(0 if passed else 1)
526
527
528if __name__ == '__main__':
529 main()