1#!/usr/bin/env python3
2"""Edit disposable notebook copies and check every changed document object."""
3import argparse
4import copy
5import hashlib
6import json
7from pathlib import Path
8import shutil
9import stat
10import subprocess
11import tempfile
12
13from document_model import DEFAULT_CONTEXT, EXPORTER, ordered_pages, view, walk
14
15ROOT = Path(__file__).resolve().parent.parent
16EDITOR = ROOT / 'target/debug/examples/random_edit'
17
18
19def export(path):
20 with tempfile.TemporaryDirectory() as temporary:
21 output = Path(temporary) / 'model'
22 subprocess.run([EXPORTER, path, output], check=True)
23 return (json.loads((output / 'document.json').read_text()),
24 json.loads((output / 'text.json').read_text()),
25 sorted(hashlib.sha256(p.read_bytes()).hexdigest() for p in (output / 'assets').iterdir()))
26
27
28def verify(before, after, record, *, author="Random editor"):
29 old, old_text, old_assets = before
30 new, new_text, new_assets = after
31 assert old_assets == new_assets, 'An edit changed embedded payload bytes'
32 sid, oid = record['space'], record['object']
33 old_rid, previous = view(old, sid)
34 new_rid, current = view(new, sid)
35 expected_runs = copy.deepcopy(old_text[sid][old_rid][oid])
36 runs = previous['nodes'][oid]['kind']['runs']
37 start, end = record['range']
38 selected = max(i for i, run in enumerate(runs) if run['start'] <= start and end <= run['end'])
39 offset = runs[selected]['start']
40 assert record['run_start'] == offset and record['run_before'] == expected_runs[selected]['text'], 'The CLI logged another insertion run'
41 encoded = expected_runs[selected]['text'].encode('utf-16-le')
42 prefix, suffix = encoded[:(start - offset) * 2], encoded[(end - offset) * 2:]
43 expected_runs[selected]['text'] = prefix.decode('utf-16-le') + record['replacement'] + suffix.decode('utf-16-le')
44 assert new_text[sid][new_rid][oid] == expected_runs, 'Text or inherited run formatting differs from the intended splice'
45 expected = copy.deepcopy(old)
46 space = expected['spaces'][sid]
47 space['contexts'][DEFAULT_CONTEXT] = new_rid
48 space['revisions'][new_rid] = copy.deepcopy(previous)
49 if old_rid not in space['contexts'].values(): del space['revisions'][old_rid]
50 node = space['revisions'][new_rid]['nodes'][oid]
51 encoded = node['kind']['text'].encode('utf-16-le')
52 node['kind']['text'] = encoded[:start * 2].decode('utf-16-le') + record['replacement'] + encoded[end * 2:].decode('utf-16-le')
53 revision = space['revisions'][new_rid]
54 page = revision['nodes'][record['page']]
55 titles = [(key, n['kind']['text']) for key, n in walk(revision, record['page'])
56 if n['kind']['type'] == 'RichText' and not n['kind']['boilerplate']
57 and any(field['id'] == 0x88001cb4 for field in n['extra'][0])]
58 assert len(titles) <= 1
59 title = titles[0][1].lstrip().split('\r')[0] if titles else ''
60 if not title or titles[0][0] == oid:
61 automatic = not title
62 if automatic:
63 body = copy.deepcopy(revision) if page['kind']['rtl'] else revision
64 if page['kind']['rtl']:
65 for item in body['nodes'].values():
66 if item['kind']['type'] == 'Row': item['children'].reverse()
67 roots = sorted(page['children'], key=lambda key: (body['nodes'][key]['layout']['y'] or 0,
68 -(body['nodes'][key]['layout']['x'] or 0)
69 if page['kind']['rtl'] else body['nodes'][key]['layout']['x'] or 0))
70 candidates = [n['kind']['text'].strip().split('\r')[0].rstrip()
71 for root in roots for _, n in walk(body, root)
72 if n['kind']['type'] == 'RichText' and not n['kind']['boilerplate']]
73 title = next((text for text in candidates if text), '')
74 encoded_title = title.encode('utf-16-le')
75 end = 510
76 if len(encoded_title) > end and 0xd800 <= int.from_bytes(encoded_title[end - 2:end], 'little') <= 0xdbff: end += 2
77 title = encoded_title[:end].decode('utf-16-le').rstrip()
78 revision['nodes'][revision['roots']['2']]['kind']['title'] = title
79 page['kind']['alternate_title'] = title if automatic else ''
80 observed = copy.deepcopy(new)
81 changed = observed['spaces'][sid]['revisions'][new_rid]['nodes'][oid]
82 assert record['started_ms'] // 1000 - 315532800 <= changed['modified'] <= record['finished_ms'] // 1000 - 315532800, 'Modification time falls outside the edit interval'
83 pending = [(record['page'], [])]
84 ancestors = set()
85 while pending:
86 key, path = pending.pop()
87 assert key not in path, 'Page content has a cycle'
88 if key == oid:
89 ancestors.update(path)
90 else:
91 parent = previous['nodes'][key]
92 pending.extend((child, path + [key]) for child in parent['children'] + parent['content'] + parent['structure'])
93 assert '4' in current['roots'], 'An edit omitted revision attribution'
94 metadata_id = current['roots']['4']
95 metadata = current['nodes'][metadata_id]
96 assert metadata['kind']['type'] == 'RevisionMetadata', 'Revision attribution has another object kind'
97 filetime = metadata['kind']['modified_filetime']
98 assert record['started_ms'] * 10000 + 116444736000000000 <= filetime < (record['finished_ms'] + 1) * 10000 + 116444736000000000, 'Revision time falls outside the edit interval'
99 author_id = metadata['latest_author']
100 assert author_id in current['nodes'], 'Revision attribution has no author object'
101 blank = {
102 'children': [], 'content': [], 'structure': [], 'spaces': [],
103 'child_level': None, 'layout': {key: None for key in node['layout']},
104 'format': {key: None for key in node['format']}, 'created': None, 'modified': None,
105 'original_author': None, 'latest_author': None, 'media_ids': [],
106 'media_time_ms': None, 'tags': [], 'extra': [[]],
107 }
108 existing_authors = {key for key, value in previous['nodes'].items()
109 if value['kind'] == {'type': 'Author', 'name': author}}
110 assert not existing_authors or author_id in existing_authors, 'An edit duplicated an existing author'
111 if author_id not in previous['nodes']:
112 initials = ''.join(word[0].upper() for word in author.split())
113 revision['nodes'][author_id] = {**copy.deepcopy(blank), 'jcid': 0x120001,
114 'kind': {'type': 'Author', 'name': author},
115 'extra': [[{'id': 0x1c001df8, 'value': {'Bytes': (initials + '\0').encode('utf-16-le').hex()}}]]}
116 assert current['nodes'][author_id]['kind'] == {'type': 'Author', 'name': author}, 'The edit names another author'
117 if '4' in previous['roots']:
118 assert metadata_id == previous['roots']['4'], 'An edit replaced existing revision metadata identity'
119 else:
120 assert metadata_id not in previous['nodes'], 'Revision metadata reused another object identity'
121 revision['roots']['4'] = metadata_id
122 revision['nodes'][metadata_id] = {**copy.deepcopy(blank), 'jcid': 0x20044,
123 'kind': {'type': 'RevisionMetadata', 'modified_filetime': None}}
124 detached_authors = {revision['nodes'][metadata_id]['latest_author']}
125 revision['nodes'][metadata_id]['latest_author'] = author_id
126 revision['nodes'][metadata_id]['kind']['modified_filetime'] = filetime
127 for key in ancestors:
128 if previous['nodes'][key]['modified'] is not None:
129 revision['nodes'][key]['modified'] = changed['modified']
130 if previous['nodes'][key]['kind']['type'] == 'Paragraph':
131 detached_authors.add(revision['nodes'][key]['latest_author'])
132 revision['nodes'][key]['latest_author'] = author_id
133 revision['nodes'][key]['modified'] = filetime // 10000000 - 11960006400
134 for key in detached_authors - {None, author_id}:
135 assert revision['nodes'][key]['kind']['type'] == 'Author'
136 if not any(json.dumps(key) in json.dumps(value) for other, value in revision['nodes'].items() if other != key):
137 del revision['nodes'][key]
138 del node['modified'], changed['modified']
139 del node['kind']['runs'], changed['kind']['runs']
140 assert expected == observed, 'An edit changed unrelated document structure, metadata or properties'
141
142
143def run(source, output, seed, rounds):
144 source = source.resolve(strict=True)
145 output = output.resolve()
146 if output.is_relative_to(source): raise ValueError('Choose an output directory outside the source notebook.')
147 output.mkdir(parents=True, exist_ok=False)
148 notebook = output / 'notebook'
149 shutil.copytree(source, notebook)
150 for file in notebook.rglob("*"):
151 if file.is_file(): file.chmod(file.stat().st_mode | stat.S_IWUSR)
152 hashes = {p.relative_to(source).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
153 for p in source.rglob('*') if p.is_file()}
154 (output / 'source.json').write_text(json.dumps(hashes, indent=2))
155 (output / 'run.json').write_text(json.dumps({'seed': seed, 'rounds': rounds,
156 'editor_sha256': hashlib.sha256(EDITOR.read_bytes()).hexdigest(),
157 'oracle_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest()}, indent=2))
158 count = 0
159 with (output / 'operations.jsonl').open('w') as log:
160 for file in sorted(notebook.rglob('*.one')):
161 before = export(file)
162 pages = [(sid, page) for sid, _, _, page in ordered_pages(before[0])]
163 for _ in range(rounds):
164 for sid, page in pages:
165 result = subprocess.run([EDITOR, file, '--in-place', str(seed + count), page], capture_output=True, text=True)
166 record = json.loads(result.stdout) if result.stdout else {}
167 record.update(file=file.relative_to(notebook).as_posix(), exit=result.returncode, stderr=result.stderr)
168 log.write(json.dumps(record, ensure_ascii=False) + '\n'); log.flush()
169 assert result.returncode == 0 and record.get('state') == 'Committed', f'Page edit failed: {record}'
170 after = export(file)
171 verify(before, after, record)
172 before = after
173 count += 1
174 assert all(hashlib.sha256((source / path).read_bytes()).hexdigest() == digest for path, digest in hashes.items()), 'A source file changed during the campaign'
175 (output / 'result.json').write_text(json.dumps({'verified_edits': count, 'source_files_unchanged': len(hashes)}, indent=2))
176 print(f'Verified {count} edits; {len(hashes)} source files unchanged')
177
178
179if __name__ == '__main__':
180 parser = argparse.ArgumentParser(description=__doc__)
181 parser.add_argument('source', type=Path)
182 parser.add_argument('output', type=Path)
183 parser.add_argument('--seed', type=int, default=42)
184 parser.add_argument('--rounds', type=int, default=1)
185 args = parser.parse_args()
186 if args.rounds <= 0 or args.seed < 0: parser.error('Use positive rounds and a nonnegative seed.')
187 run(args.source, args.output, args.seed, args.rounds)