1#[path = "support/ops.rs"]
2mod ops;
3use onestore::{
4 ExGuid, FileDataReference, IdStream, ObjectData, PropertySets, RevisionIndex, Store, Value,
5 document::Document,
6};
7use std::collections::{BTreeMap, BTreeSet};
8
9fn compare_copy(left: &Store<'_>, right: &Store<'_>, source: ExGuid, copied: ExGuid) {
10 let indexes = [left, right].map(|store| RevisionIndex::parse(store).unwrap());
11 let mut contexts = vec![(source, ExGuid::default(), copied, ExGuid::default())];
12 let mut mapped_contexts = BTreeMap::new();
13 let mut inverse_contexts = BTreeMap::new();
14 while let Some((ls, lc, rs, rc)) = contexts.pop() {
15 if let Some(previous) = mapped_contexts.insert((ls, lc), (rs, rc)) {
16 assert_eq!(previous, (rs, rc));
17 continue;
18 }
19 assert_eq!(inverse_contexts.insert((rs, rc), (ls, lc)), None);
20 let revisions =
21 [(ls, lc, &indexes[0]), (rs, rc, &indexes[1])].map(|(sid, context, index)| {
22 index
23 .resolve(sid, index.spaces[&sid].labels[&(context, 1)])
24 .unwrap()
25 });
26 assert_eq!(
27 revisions[0].roots.keys().collect::<Vec<_>>(),
28 revisions[1].roots.keys().collect::<Vec<_>>()
29 );
30 let manifest = &revisions[1].objects[&revisions[1].roots[&1]];
31 let copied_time = if manifest.jcid == 0x60037 {
32 let pages = manifest.references().unwrap().objects;
33 assert_eq!(pages.len(), 1);
34 let ObjectData::Properties(bytes) = revisions[1].objects[&pages[0]].data else {
35 panic!()
36 };
37 let properties = PropertySets::parse(bytes).unwrap();
38 let Value::Bytes(time) = properties.sets[0]
39 .iter()
40 .find(|p| p.id == 0x14001d7a)
41 .unwrap()
42 .value
43 else {
44 panic!()
45 };
46 Some(time)
47 } else {
48 None
49 };
50 let mut pending: Vec<_> = revisions[0]
51 .roots
52 .iter()
53 .map(|(role, id)| (*id, revisions[1].roots[role]))
54 .collect();
55 let mut mapped = BTreeMap::new();
56 let mut inverse = BTreeMap::new();
57 while let Some((a, b)) = pending.pop() {
58 if let Some(previous) = mapped.insert(a, b) {
59 assert_eq!(previous, b);
60 continue;
61 }
62 if let Some(previous) = inverse.insert(b, a) {
63 let originals = [&revisions[0].objects[&previous], &revisions[0].objects[&a]];
64 assert!(originals.iter().all(|object| object.jcid == 0x120001));
65 assert_eq!(originals[0].data, originals[1].data);
66 for object in originals {
67 let references = object.references().unwrap();
68 assert!(references.objects.is_empty());
69 assert!(references.object_spaces.is_empty());
70 assert!(references.contexts.is_empty());
71 }
72 }
73 let objects = [&revisions[0].objects[&a], &revisions[1].objects[&b]];
74 assert_eq!(objects[0].jcid, objects[1].jcid);
75 match (objects[0].data, objects[1].data) {
76 (ObjectData::Properties(a), ObjectData::Properties(b)) => {
77 let sets = [a, b].map(|bytes| PropertySets::parse(bytes).unwrap());
78 let mut nested = vec![(0, 0)];
79 while let Some((a, b)) = nested.pop() {
80 let fields = [(0, a), (1, b)].map(|(side, set)| {
81 let properties = &sets[side].sets[set];
82 let fields = properties
83 .iter()
84 .map(|p| (p.id, &p.value))
85 .collect::<BTreeMap<_, _>>();
86 assert_eq!(fields.len(), properties.len());
87 fields
88 });
89 assert_eq!(
90 fields[0].keys().collect::<Vec<_>>(),
91 fields[1].keys().collect::<Vec<_>>()
92 );
93 for (id, value) in &fields[0] {
94 let other = fields[1][id];
95 match (value, other) {
96 (
97 Value::References {
98 stream,
99 compact_ids,
100 },
101 Value::References {
102 stream: other_stream,
103 compact_ids: other_ids,
104 },
105 ) => {
106 assert_eq!(stream, other_stream);
107 assert_eq!(compact_ids.len(), other_ids.len());
108 for (a, b) in compact_ids
109 .as_chunks::<4>()
110 .0
111 .iter()
112 .zip(other_ids.as_chunks::<4>().0.iter())
113 {
114 let ids = [a, b].map(|bytes| u32::from_le_bytes(*bytes));
115 let [a, b] = [0, 1].map(|side| ExGuid {
116 guid: objects[side].global_ids[&(ids[side] >> 8)],
117 n: ids[side] & 255,
118 });
119 match stream {
120 IdStream::Objects => pending.push((a, b)),
121 IdStream::Contexts => contexts.push((ls, a, rs, b)),
122 IdStream::ObjectSpaces => contexts.push((
123 a,
124 ExGuid::default(),
125 b,
126 ExGuid::default(),
127 )),
128 }
129 }
130 }
131 (Value::Sets(a), Value::Sets(b)) => {
132 assert_eq!(a.len(), b.len());
133 nested.extend(a.clone().zip(b.clone()));
134 }
135 (Value::Bytes(_), Value::Bytes(level))
136 if objects[0].jcid == 0x20030
137 && *id == 0x14001dff
138 && lc == ExGuid::default() =>
139 {
140 assert_eq!(*level, 1_u32.to_le_bytes());
141 }
142 (Value::Bytes(_), Value::Bytes(time))
143 if matches!(
144 objects[0].jcid,
145 0x6000b
146 | 0x6000c
147 | 0x6002c
148 | 0x60022
149 | 0x60012
150 | 0x60023
151 | 0x60024
152 ) && *id == 0x14001d7a
153 && lc == ExGuid::default() =>
154 {
155 if value != &other {
156 assert_eq!(Some(*time), copied_time);
157 }
158 }
159 _ => assert_eq!(
160 value, &other,
161 "property {id:#x}, jcid {:#x}, context {lc}, source set {a}, copied set {b}",
162 objects[0].jcid
163 ),
164 }
165 }
166 }
167 }
168 (ObjectData::File { extension: a, .. }, ObjectData::File { extension: b, .. }) => {
169 assert_eq!(a, b);
170 let references =
171 objects.map(|object| object.file_reference().unwrap().unwrap());
172 let [
173 FileDataReference::Internal(a),
174 FileDataReference::Internal(b),
175 ] = references
176 else {
177 panic!("The copy fixture requires its external payloads");
178 };
179 assert_eq!(left.file_data(a).unwrap(), right.file_data(b).unwrap());
180 }
181 _ => panic!("Copied object representations differ"),
182 }
183 }
184 }
185}
186
187#[test]
188fn native_parent_removal_retains_history_and_copies_the_complete_page() {
189 let bytes = [
190 include_bytes!("../../../corpus/page-lifecycle/05-reordered/notebook/Lifecycle.one").as_slice(),
191 include_bytes!("../../../corpus/page-lifecycle/06-deleted-parent/notebook/Lifecycle.one").as_slice(),
192 include_bytes!("../../../corpus/page-lifecycle/06-deleted-parent/notebook/OneNote_RecycleBin/OneNote_DeletedPages.one").as_slice(),
193 ];
194 let stores = bytes.map(|b| Store::parse(b).unwrap());
195 let indexes = stores.each_ref().map(|s| RevisionIndex::parse(s).unwrap());
196 let documents = indexes.each_ref().map(|i| Document::parse(i).unwrap());
197 let sid = documents[0].pages().unwrap()[4].0;
198 assert!(!documents[1].spaces.contains_key(&sid));
199 assert_eq!(
200 indexes[0].spaces.keys().collect::<Vec<_>>(),
201 indexes[1].spaces.keys().collect::<Vec<_>>()
202 );
203 for (space, revisions) in &indexes[0].spaces {
204 for rid in revisions.revisions.keys() {
205 assert_eq!(
206 format!("{:?}", indexes[0].resolve(*space, *rid).unwrap()),
207 format!("{:?}", indexes[1].resolve(*space, *rid).unwrap())
208 );
209 }
210 }
211 let current = indexes[1]
212 .resolve(sid, indexes[1].spaces[&sid].labels[&(ExGuid::default(), 1)])
213 .unwrap();
214 let ObjectData::Properties(manifest) = current.objects[&current.roots[&1]].data else {
215 panic!()
216 };
217 assert!(PropertySets::parse(manifest).unwrap().sets[0].is_empty());
218 let ObjectData::Properties(metadata) = current.objects[&current.roots[&2]].data else {
219 panic!()
220 };
221 assert!(
222 PropertySets::parse(metadata).unwrap().sets[0]
223 .iter()
224 .any(|p| p.id == 0x88001de9)
225 );
226 let copied = documents[2].pages().unwrap();
227 assert_eq!(copied.len(), 1);
228 compare_copy(&stores[0], &stores[2], sid, copied[0].0);
229}
230
231fn compare_labels(before: &RevisionIndex<'_>, after: &RevisionIndex<'_>) {
232 assert_eq!(
233 before.spaces.keys().collect::<Vec<_>>(),
234 after.spaces.keys().collect::<Vec<_>>()
235 );
236 for (sid, space) in &before.spaces {
237 assert_eq!(
238 space.labels.keys().collect::<Vec<_>>(),
239 after.spaces[sid].labels.keys().collect::<Vec<_>>()
240 );
241 for (label, rid) in &space.labels {
242 assert_eq!(
243 format!("{:?}", before.resolve(*sid, *rid).unwrap()),
244 format!(
245 "{:?}",
246 after
247 .resolve(*sid, after.spaces[sid].labels[label])
248 .unwrap()
249 ),
250 "cold context {sid}/{label:?}"
251 );
252 }
253 }
254}
255
256fn native_case(name: &str) {
257 let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
258 .join("../../corpus/page-lifecycle/removal")
259 .join(name);
260 let request: serde_json::Value =
261 serde_json::from_slice(&std::fs::read(root.join("removal.json")).unwrap()).unwrap();
262 let selected: BTreeSet<usize> = serde_json::from_value(request["selected"].clone()).unwrap();
263 let bytes = ["native/before", "native/after", "cold"]
264 .map(|phase| std::fs::read(root.join(phase).join("notebook/Lifecycle.one")).unwrap());
265 let stores = bytes.each_ref().map(|bytes| Store::parse(bytes).unwrap());
266 let indexes = stores.each_ref().map(|store| {
267 assert!(store.checksum_mismatches.is_empty());
268 let index = RevisionIndex::parse(store).unwrap();
269 index.validate_current().unwrap();
270 index
271 });
272 let documents = indexes
273 .each_ref()
274 .map(|index| Document::parse(index).unwrap());
275 let before = documents[0].pages().unwrap();
276 let retained: Vec<_> = before
277 .iter()
278 .enumerate()
279 .filter(|(index, _)| !selected.contains(index))
280 .map(|(_, page)| *page)
281 .collect();
282 assert_eq!(documents[1].pages().unwrap(), retained);
283 assert_eq!(documents[2].pages().unwrap(), retained);
284 let groups = documents.each_ref().map(|document| {
285 let space = &document.spaces[&document.root];
286 let revision = &space.revisions[&space.contexts[&ExGuid::default()]];
287 revision.nodes[&revision.roots[&1]]
288 .children
289 .iter()
290 .map(|id| (*id, revision.nodes[id].spaces.clone()))
291 .collect::<Vec<_>>()
292 });
293 let mut expected: Vec<(ExGuid, Vec<ExGuid>)> = Vec::new();
294 for (sid, _) in &retained {
295 let space = &documents[0].spaces[sid];
296 let revision = &space.revisions[&space.contexts[&ExGuid::default()]];
297 let onestore::document::Kind::Metadata { level, .. } =
298 revision.nodes[&revision.roots[&2]].kind
299 else {
300 panic!()
301 };
302 if expected.is_empty() || level == Some(1) {
303 let (series, _) = groups[0]
304 .iter()
305 .find(|(_, pages)| pages.contains(sid))
306 .unwrap();
307 expected.push((*series, Vec::new()));
308 }
309 expected.last_mut().unwrap().1.push(*sid);
310 }
311 assert_eq!(groups[1], expected);
312 assert_eq!(groups[2], expected);
313 for (phase, index) in indexes[1..].iter().enumerate() {
314 assert_eq!(
315 indexes[0].spaces.keys().collect::<Vec<_>>(),
316 index.spaces.keys().collect::<Vec<_>>()
317 );
318 let mut discarded = 0;
319 for (sid, space) in &indexes[0].spaces {
320 for rid in space.revisions.keys() {
321 if !index.spaces[sid].revisions.contains_key(rid) {
322 discarded += 1;
323 continue;
324 }
325 assert_eq!(
326 format!("{:?}", indexes[0].resolve(*sid, *rid).unwrap()),
327 format!("{:?}", index.resolve(*sid, *rid).unwrap()),
328 "historical revision {sid}/{rid}"
329 );
330 }
331 for ((context, role), rid) in &space.labels {
332 if *context == ExGuid::default()
333 || selected.iter().any(|ordinal| before[*ordinal].0 == *sid)
334 {
335 continue;
336 }
337 let retained = index.spaces[sid].labels[&(*context, *role)];
338 assert_eq!(
339 format!("{:?}", indexes[0].resolve(*sid, *rid).unwrap()),
340 format!("{:?}", index.resolve(*sid, retained).unwrap()),
341 "labeled history {sid}/{context}/{role}"
342 );
343 }
344 }
345 assert_eq!(
346 discarded,
347 match name {
348 "ink" => 9,
349 "features" => 8,
350 _ => 0,
351 },
352 "native compaction phase {phase}"
353 );
354 }
355 compare_labels(&indexes[1], &indexes[2]);
356 let active = |side: usize, sid| {
357 indexes[side]
358 .resolve(
359 sid,
360 indexes[side].spaces[&sid].labels[&(ExGuid::default(), 1)],
361 )
362 .unwrap()
363 };
364 for (ordinal, (sid, _)) in before.iter().enumerate() {
365 let old = active(0, *sid);
366 let after = active(1, *sid);
367 assert_eq!(old.roots, after.roots);
368 if selected.contains(&ordinal) {
369 let ObjectData::Properties(manifest) = after.objects[&after.roots[&1]].data else {
370 panic!()
371 };
372 assert!(PropertySets::parse(manifest).unwrap().sets[0].is_empty());
373 let ObjectData::Properties(metadata) = after.objects[&after.roots[&2]].data else {
374 panic!()
375 };
376 assert!(
377 PropertySets::parse(metadata).unwrap().sets[0]
378 .iter()
379 .any(|p| p.id == 0x88001de9)
380 );
381 } else {
382 assert_eq!(
383 old.objects.keys().collect::<Vec<_>>(),
384 after.objects.keys().collect::<Vec<_>>()
385 );
386 for (id, object) in &old.objects {
387 if *id != old.roots[&2] {
388 assert_eq!(
389 format!("{object:?}"),
390 format!("{:?}", after.objects[id]),
391 "survivor {sid}/{id}"
392 );
393 } else {
394 let fields = [object, &after.objects[id]].map(|object| {
395 let ObjectData::Properties(bytes) = object.data else {
396 panic!()
397 };
398 PropertySets::parse(bytes).unwrap().sets[0]
399 .iter()
400 .filter(|p| p.id != 0x14001dff)
401 .map(|p| (p.id, format!("{:?}", p.value)))
402 .collect::<BTreeMap<_, _>>()
403 });
404 assert_eq!(fields[0], fields[1]);
405 use onestore::document::Kind;
406 let levels = [0, 1].map(|side| {
407 let space = &documents[side].spaces[sid];
408 let view = &space.revisions[&space.contexts[&ExGuid::default()]];
409 let Kind::Metadata { level, .. } = view.nodes[id].kind else {
410 panic!()
411 };
412 level
413 });
414 assert_eq!(
415 levels[1],
416 if retained.first().map(|p| p.0) == Some(*sid) {
417 Some(1)
418 } else {
419 levels[0]
420 }
421 );
422 }
423 }
424 }
425 }
426 if request["permanent"].as_bool().unwrap() {
427 assert!(
428 !root
429 .join("native/after/notebook/OneNote_RecycleBin")
430 .exists()
431 );
432 return;
433 }
434 let recycle_bytes = std::fs::read(
435 root.join("native/after/notebook/OneNote_RecycleBin/OneNote_DeletedPages.one"),
436 )
437 .unwrap();
438 let recycle = Store::parse(&recycle_bytes).unwrap();
439 let recycled_index = RevisionIndex::parse(&recycle).unwrap();
440 recycled_index.validate_current().unwrap();
441 let recycled_document = Document::parse(&recycled_index).unwrap();
442 let cold_recycle_bytes =
443 std::fs::read(root.join("cold/notebook/OneNote_RecycleBin/OneNote_DeletedPages.one"))
444 .unwrap();
445 let cold_recycle = Store::parse(&cold_recycle_bytes).unwrap();
446 let cold_index = RevisionIndex::parse(&cold_recycle).unwrap();
447 cold_index.validate_current().unwrap();
448 compare_labels(&recycled_index, &cold_index);
449 let pages = recycled_document.pages().unwrap();
450 assert_eq!(pages.len(), selected.len());
451 let identity = |index: &RevisionIndex<'_>, sid| {
452 let revision = index
453 .resolve(sid, index.spaces[&sid].labels[&(ExGuid::default(), 1)])
454 .unwrap();
455 let ObjectData::Properties(bytes) = revision.objects[&revision.roots[&2]].data else {
456 panic!()
457 };
458 let sets = PropertySets::parse(bytes).unwrap();
459 let Value::Bytes(guid) = sets.sets[0]
460 .iter()
461 .find(|p| p.id == 0x1c001c30)
462 .unwrap()
463 .value
464 else {
465 panic!()
466 };
467 guid.to_vec()
468 };
469 let mut copies = BTreeMap::new();
470 for (sid, _) in pages {
471 assert!(copies.insert(identity(&recycled_index, sid), sid).is_none());
472 }
473 for ordinal in selected {
474 let sid = before[ordinal].0;
475 let copied = copies.remove(&identity(&indexes[0], sid)).unwrap();
476 compare_copy(&stores[0], &recycle, sid, copied);
477 compare_copy(&stores[0], &cold_recycle, sid, copied);
478 }
479 assert!(copies.is_empty());
480}
481
482macro_rules! native_cases {
483 ($($name:ident => $case:literal),* $(,)?) => {$(
484 #[test]
485 fn $name() { native_case($case); }
486 )*};
487}
488
489native_cases! {
490 trailing => "trailing", parent => "parent", child => "child", grandchild => "grandchild",
491 group => "group", all => "all", permanent_parent => "permanent-parent",
492 leading_parent => "leading-parent", leading_group => "leading-group",
493 features => "features", ink => "ink",
494}
495
496#[test]
497fn copy_comparison_rejects_text_and_payload_changes() {
498 let original = include_bytes!(
499 "../../../corpus/page-lifecycle/removal/ink/native/before/notebook/Lifecycle.one"
500 );
501 let copied = include_bytes!(
502 "../../../corpus/page-lifecycle/removal/ink/native/after/notebook/OneNote_RecycleBin/OneNote_DeletedPages.one"
503 );
504 let stores = [original.as_slice(), copied.as_slice()].map(|bytes| Store::parse(bytes).unwrap());
505 let indexes = stores
506 .each_ref()
507 .map(|store| RevisionIndex::parse(store).unwrap());
508 let documents = indexes
509 .each_ref()
510 .map(|index| Document::parse(index).unwrap());
511 let [source_sid, copy_sid] = documents
512 .each_ref()
513 .map(|document| document.pages().unwrap()[0].0);
514 compare_copy(&stores[0], &stores[1], source_sid, copy_sid);
515 let space = &documents[1].spaces[&copy_sid];
516 let revision = &space.revisions[&space.contexts[&ExGuid::default()]];
517 let text = revision
518 .nodes
519 .iter()
520 .find_map(|(oid, node)| {
521 matches!(
522 node.kind,
523 onestore::document::Kind::RichText {
524 boilerplate: false,
525 ..
526 }
527 )
528 .then_some(*oid)
529 })
530 .unwrap();
531 let changed = ops::page_edited(
532 copied,
533 copy_sid,
534 vec![onestore::op::PageOp::Text {
535 text,
536 range: 0..0,
537 with: "Changed ".into(),
538 }],
539 )
540 .unwrap();
541 let changed_store = Store::parse(&changed).unwrap();
542 RevisionIndex::parse(&changed_store)
543 .unwrap()
544 .validate_current()
545 .unwrap();
546 assert!(
547 std::panic::catch_unwind(|| compare_copy(&stores[0], &changed_store, source_sid, copy_sid))
548 .is_err()
549 );
550
551 let raw = indexes[1]
552 .resolve(
553 copy_sid,
554 indexes[1].spaces[&copy_sid].labels[&(ExGuid::default(), 1)],
555 )
556 .unwrap();
557 let attachment = revision
558 .nodes
559 .values()
560 .find_map(|node| match node.kind {
561 onestore::document::Kind::Attachment { container, .. } => container,
562 _ => None,
563 })
564 .unwrap();
565 let Some(FileDataReference::Internal(guid)) =
566 raw.objects[&attachment].file_reference().unwrap()
567 else {
568 panic!()
569 };
570 let payload = stores[1].file_data(guid).unwrap();
571 assert!(!payload.is_empty());
572 let offset = payload.as_ptr().addr() - copied.as_ptr().addr();
573 let mut changed = copied.to_vec();
574 changed[offset] ^= 1;
575 let changed_store = Store::parse(&changed).unwrap();
576 assert!(changed_store.checksum_mismatches.is_empty());
577 RevisionIndex::parse(&changed_store)
578 .unwrap()
579 .validate_current()
580 .unwrap();
581 assert_ne!(changed_store.file_data(guid).unwrap(), payload);
582 assert!(
583 std::panic::catch_unwind(|| compare_copy(&stores[0], &changed_store, source_sid, copy_sid))
584 .is_err()
585 );
586}