1// @effect-diagnostics globalDate:off globalTimers:off - Vendored @clo/lib has no Effect dependency.
2/**
3 * a progress reporting API that allows creating CLI spinners and progress bars
4 * that can be easily nested to visualize complex, parallel work. in addition
5 * to providing rich feedback in the terminal, a headless {@linkcode Root} node
6 * can be constructed, which can be bound to the DOM through `progress/browser`
7 * or transmit progress information
8 * {@link encodeEventStream|across a network}.
9 *
10 * the progress API takes advantage of the `using` keyword to make nodes
11 * scoped. the convention is to make functions take a {@linkcode Ref|progress.Ref},
12 * to act as the destination for the sub-node.
13 *
14 * ```ts
15 * async function doSomething(p: progress.Ref) {
16 * using node = p.start("my task");
17 * // ...
18 * await doSomethingElse(node); // node satisfies progress.Ref
19 * // ...
20 * }
21 * async function doSomethingElse(p: progress.Ref) {
22 * using node = p.start("my sub task");
23 * // ...
24 * }
25 *
26 * doSomething(progress); // the progress module satisfies progress.Ref
27 *
28 * import * as progress from "@clo/lib/progress";
29 * ```
30 *
31 * in some cases, such as in `subprocess/ffmpeg.ts`, it may make more sense
32 * to pass in a pre-started node. the `ffmpeg` module, for example, uses the
33 * starting text as a base title and decorates it with encoding stats.
34 *
35 * when using the top-level {@linkcode start|progress.start}, proper
36 * integration is made with `lib/log.ts` so that the progress tree does not
37 * intersect with regular log messages, as well as forwarding each
38 * {@linkcode Node} custom logging scope to the global log, including any
39 * custom redirections.
40 *
41 * this module is under construction. while i am happy with the overall API, it
42 * needs more work and feature development. the API of `Node` is stable, though.
43 *
44 * inspired by the [Zig Progress API](https://andrewkelley.me/post/zig-new-cli-progress-bar-explained.html).
45 *
46 * @module
47 */
48
49/**
50 * creates a new trackable unit of work, visible in the terminal as a spinner
51 * that rests at the end of the log. if `estimate` is given, a progress bar is
52 * shown.
53 */
54/* node:coverage ignore next 3 */
55export function start(text: string, opts?: StartOptions): Node {
56 return global.start(text, opts);
57}
58
59/**
60 * a reference point to create sub-items. this interface is satisfied by
61 * - the `progress` module itself
62 * - a node recieved by `p.start("label")`
63 * - a headless progress runner from `progress.headless()`
64 *
65 * prefer taking `Ref` as a function parameter, so that the function can be
66 * started at the top level, or as a sub-task of an existing node.
67 *
68 * ```ts
69 * function build(p: progress.Ref = progress) {
70 * using _ = p.start("build process");
71 * // ...
72 * }
73 * ```
74 *
75 * sometimes it makes sense to have an API recieve `Node` instead of `Ref`,
76 * specifically if the caller should have some control over the node name.
77 *
78 * ```ts
79 * await ffmpeg.spawn({
80 * cmd: ["-i", "hello.mov", "-c:v", "svtav1", "hello.mp4"],
81 * progress: progress.start("encode hello.mov"),
82 * });
83 * ```
84 */
85export interface Ref {
86 /**
87 * creates a new trackable unit of work as a child of this one.
88 * when given an estimate, a progress bar is rendered.
89 */
90 start(text: string, opts?: StartOptions): Node;
91}
92
93/**
94 * control for a progress nooe. all fields are primed with setters to trigger
95 * ui dispatch automatically. to make tools that use `progress.ts` more
96 * modular to a custom progress implementation, many fields are optional.
97 */
98export interface Node extends Ref, Disposable {
99 /** end this Node or RootNode. ending collapses children. */
100 end(this: Node): void;
101 /** one line of text display. */
102 text: string;
103 /**
104 * number of items completed. by default this is a count of items, but the
105 * `units` property can be passed to `start` to change how the `value` is
106 * interpretted when displaying this progress node.
107 */
108 value: number;
109 /**
110 * an estimate of how many items this task contains.
111 * when set above `0`, it is shown with a progress bar.
112 */
113 total: number;
114 /**
115 * defaults to increasing by 1. if incrementing causes `value >= estimate`,
116 * it also calls `end`. use `node.value += 1` if that behavior is not wanted.
117 */
118 inc(this: Node, value?: number): void;
119 /** a scoped logger set to output inline on this item. */
120 readonly log: log.RootScope;
121
122 /**
123 * a time for when this node is estimated to be completed, in milliseconds
124 * relative to the unix epoch. by default, this is automatically managed and
125 * made read-only. if `StartOptions` specifies `estimateCompletions: false`,
126 * this field may be mutated.
127 */
128 estimatedTime?: number | null | undefined;
129 /**
130 * default `true`. `false` will hides a bar + printing total. this is useful
131 * for preseving auto-end behavior of `inc` without showing the total.
132 */
133 showTotal?: boolean;
134 /**
135 * default `false`. when true, this progress item is hidden if
136 * there are no children.
137 */
138 passive?: boolean;
139 /** default `false`. when true, this progress item is hidden */
140 hidden?: boolean;
141 /** override sorting for children */
142 sortChildren?: ((a: ReadOnlyNode, b: ReadOnlyNode) => number) | null;
143 /**
144 * specify how `value` and `total` is formed into text.
145 * this has some discoverable presets on `StartOptions.units`.
146 */
147 valueFormatter?: (value: number, total: number | null) => string;
148}
149
150/** override the default field values of {@linkcode Node} on creation. */
151export interface StartOptions {
152 /** number of items already completed. */
153 value?: number | undefined | null;
154 /** when set above `0`, it is shown with a progress bar. */
155 total?: number | undefined | null;
156 /**
157 * presets for common formattings of `value` and `total`.
158 * @default "count"
159 */
160 units?: "count" | "bytes" | "percent";
161 /**
162 * customize how an estimate is given for progress bars. to make
163 * `Node.estimatedTime` mutable, pass `false` here.
164 * @default `null`, which is equivilent to `new progress.Ema()`
165 */
166 estimateCompletion?: EstimationAlgorithm | boolean | null;
167 /**
168 * when true, this progress item is hidden if there are no children.
169 * this makes sense when scheduling items with a priority queue, where a
170 * passive node is only used for grouping, but children are used for
171 * indicating status.
172 * @default false
173 */
174 passive?: boolean | undefined | null;
175 /** default `false`. when true, this progress item is hidden */
176 hidden?: boolean | undefined | null;
177 /**
178 * default `true`. `false` will hides a bar + printing estimate,
179 * but preseving auto-end behavior of `inc`
180 */
181 showTotal?: boolean | undefined | null;
182 /** specify sorting for children */
183 sortChildren?: Node["sortChildren"] | undefined | null;
184 /**
185 * specify how `value` and `total` is formed into text.
186 * this has some discoverable presets on `StartOptions.units`.
187 */
188 valueFormatter?: ValueFormatter;
189 /** pre-fill the list of log messages */
190 messages?: log.Message[] | undefined;
191}
192
193/**
194 * value` will be at least zero, total will be `null` or greater than zero. a
195 * custom value formatter can be passed to {@linkcode start} or mutated on an
196 * existing node.
197 */
198export type ValueFormatter = (value: number, total: number | null) => string;
199
200/**
201 * creates a function to use as a `Node.valueFormatter` by using a single
202 * number formatter, and then joining the node's value and total with a "/",
203 * but leaving it out if there is no total.
204 */
205export function slashValueFormatter(fmtNumber: (value: number) => string): ValueFormatter {
206 return (value: number, total: number | null) =>
207 value > 0 ? fmtNumber(value) + (total != null && total > 0 ? "/" + fmtNumber(total) : "") : "";
208}
209
210/** the formatter used when passing `units: "count"` to {@linkcode start} */
211export const defaultValueFormatter: ValueFormatter = /* @__PURE__ */ slashValueFormatter(String);
212
213/** the formatter used when passing `units: "bytes"` to {@linkcode start} */
214export const bytesValueFormatter: ValueFormatter =
215 /* @__PURE__ */
216 slashValueFormatter(string.formatByteSize);
217
218/** the formatter used when passing `units: "percent"` to {@linkcode start} */
219export function percentValueFormatter(value: number, total: number | null): string {
220 if (total != null && total > 0) {
221 value /= total;
222 }
223 return `${Math.round(value * 1000) / 10}%`;
224}
225
226/**
227 * {@linkcode Root} allows overriding it's timing functions, which are used for
228 * mostly for rendering and streaming adapters.
229 */
230export interface RootOptions {
231 delay?: typeof async.delay;
232 /** used for debouncing and UI. time estimation does NOT use this method */
233 now?: typeof performance.now;
234}
235
236/**
237 * a headless progress node. when the state of the tree changes, the `change`
238 * event is emitted (batched with many changes). the primary use case of using
239 * a custom `Root`s is to stream a task's progress over a network or process
240 * IPC using {@linkcode encodeByteStream}/{@linkcode encodeEventStream}.
241 * another use case is to render a progress task differently.
242 *
243 * custom events may be dispatched on the `Root`, but this is only useful when
244 * streaming, since the streams will carry any custom events.
245 */
246export class Root<Result = void, Map extends Events.Map = ts.EmptyObject> extends Events<
247 MergeRootEvents<Result, Map>
248> {
249 delay: typeof async.delay;
250 now: typeof performance.now;
251 #status: "active" | "end" | "error" = "active";
252 #active: Internal[] = [];
253 #debounce: async.Cancelable<void> | null = null;
254
255 constructor({ delay, now }: RootOptions = {}) {
256 super();
257 this.delay = delay ?? async.delay;
258 this.now = now ?? performance.now.bind(performance);
259 this.on("node-end", (state) => {
260 if (state.parent) return;
261 const i = this.#active.indexOf(state as Internal);
262 ASSERT(i !== -1);
263 this.#active.splice(i, 1);
264 });
265 }
266
267 /** the caller may not mutate `Root`'s state */
268 get active(): readonly ReadOnlyNode[] {
269 return this.#active;
270 }
271
272 /**
273 * creates a new trackable unit of work at the top level of this root.
274 * when given an estimate, a progress bar is rendered.
275 */
276 start(text: string, opts?: StartOptions): Node {
277 ASSERT(this.#status === "active");
278 const [state, node] = newNode(this, text, opts);
279 this.#active.push(state);
280 this.emit("node-start", state, null);
281 return node;
282 }
283
284 /**
285 * end this root, as well as all children nodes. this emits the `end` event
286 * with the provided value, JSON-serializing it if connected via
287 * {@linkcode encodeEventStream} or {@linkcode encodeByteStream}.
288 *
289 * this function is pre-bound so that it can be passed to `then`:
290 * ```ts
291 * const root = new Root();
292 * void asyncFunction().then(root.end, root.error);
293 * ```
294 */
295 end = (result: Result): void => {
296 this.emit("end", result);
297 };
298
299 /**
300 * end this root, as well as all children nodes. this emits the `error` event.
301 */
302 error = (error: unknown): void => {
303 this.emit("error", error);
304 };
305
306 /** convert this root into a promise. */
307 asPromise(): Promise<Result> {
308 return this.once("end").then((x) => x[0]);
309 }
310
311 /**
312 * emit an event on the specified channel. behavior is not defined when manually
313 * emitting one of the built in events.
314 */
315 override emit<C extends keyof MergeRootEvents<Result, Map>>(
316 channel: C,
317 ...args: MergeRootEvents<Result, Map>[C]
318 ): void {
319 ASSERT(this.#status === "active");
320 if (channel !== "change" && rootEvents.has(channel)) this.#emitChangeSoon();
321 if (channel === "error" || channel === "end") {
322 this.#end();
323 this.#status = channel;
324 }
325 super.emit(channel, ...args);
326 }
327
328 #end() {
329 const hasActive = this.#active.length > 0 || !this.#debounce;
330 for (const active of this.#active) endNode(this, active);
331 this.#active.splice(0, this.#active.length);
332 this.#debounce?.cancel();
333 if (hasActive) this.emit("change", this.#active);
334 }
335
336 #emitChangeSoon = () => {
337 if (this.#debounce) return;
338 (this.#debounce = this.delay(1)).then(() => {
339 this.#debounce = null;
340 this.emit("change", this.#active);
341 });
342 };
343}
344
345/** the built in events provided by {@linkcode Root} */
346export type RootEventMap<Result = void> = {
347 // this event is the debounced "ready to re-render" event. do not emit.
348 change: [rootNodes: readonly ReadOnlyNode[]];
349 // these events are emitted without a debounce, and are used for general
350 // communication within `Root`'s implementation. do not emit.
351 "node-start": [newNode: ReadOnlyNode, parentNode: ReadOnlyNode | null];
352 "node-change": [node: ReadOnlyNode, key: keyof ReadOnlyNode];
353 "node-end": [node: ReadOnlyNode];
354 "node-detached-log": [msg: log.Message];
355 // these events are used for result communication. they can be sent via
356 // `emit` or the shorthand methods `end` and `error`. sending either will put
357 // the `Root` into a "done" state.
358 end: [result: Result];
359 error: [error: unknown];
360};
361
362const rootEvents: ReadonlySet<unknown> = new Set([
363 "change",
364 "node-start",
365 "node-change",
366 "node-end",
367 "node-detached-log",
368 "end",
369 "error",
370]);
371
372type MergeRootEvents<Result, Map extends Events.Map> = {
373 [K in keyof Map | keyof RootEventMap]: K extends keyof RootEventMap
374 ? RootEventMap<Result>[K]
375 : Map[K];
376};
377
378/** a read-only version of {@linkcode Node}, provided to renderers. */
379export type ReadOnlyNode = Readonly<
380 Pick<
381 Required<Node>,
382 | "text"
383 | "value"
384 | "total"
385 | "estimatedTime"
386 | "passive"
387 | "hidden"
388 | "showTotal"
389 | "sortChildren"
390 | "valueFormatter"
391 > & {
392 /** unique identifier. once {@linkcode Node} is disposed, the key is reused. */
393 key: number;
394 logs: readonly log.Message[];
395 /** traverse down the tree. */
396 children: readonly ReadOnlyNode[];
397 /** traverse up the tree. */
398 parent: ReadOnlyNode | null;
399 }
400>;
401
402/** internal state is read-write. */
403interface Internal extends ts.Writeable<ReadOnlyNode> {
404 logs: log.Message[];
405 children: Internal[];
406 parent: Internal | null;
407 detached: boolean;
408}
409
410// TypeScript Test
411void function (unknown: unknown) {
412 unknown as Internal satisfies ReadOnlyNode;
413};
414
415/**
416 * constructs a linked `Node` and `Internal` that dispatches events to `owner`.
417 * the `Internal` is the non-reactive source of truth that can be blindly cast
418 * to `ReadOnlyNode`, and `Node` is the Node.
419 */
420function newNode<Result, EventMap extends Events.Map>(
421 owner: Root<Result, EventMap>,
422 text: string,
423 opts: StartOptions = {},
424): [Internal, Node] {
425 let estimator: EstimationAlgorithm | null = null;
426
427 const state: Internal = {
428 key: globalKeyPool.get(),
429 text,
430 value: opts.value ?? 0,
431 total: opts.total ?? 0,
432 estimatedTime: null,
433 showTotal: opts.showTotal ?? true,
434 passive: opts.passive ?? false,
435 hidden: opts.hidden ?? false,
436 logs: [...(opts.messages ?? [])],
437 children: [],
438 parent: null,
439 sortChildren: opts.sortChildren ?? null,
440 valueFormatter:
441 opts.valueFormatter ??
442 {
443 count: defaultValueFormatter,
444 bytes: bytesValueFormatter,
445 percent: percentValueFormatter,
446 }[opts.units ?? "count"],
447 detached: false,
448 };
449
450 if (opts.estimateCompletion !== false) {
451 const { estimateCompletion } = opts;
452 if (estimateCompletion && typeof estimateCompletion === "object") {
453 estimator = estimateCompletion;
454 } else estimator = new Ema();
455 }
456 estimator?.sample(Date.now(), 0, 0);
457
458 function mutate(key: keyof ReadOnlyNode) {
459 if (state.detached) return;
460 if (
461 key === "value" &&
462 estimator &&
463 state.total > 0 &&
464 state.value > 0 &&
465 state.value < state.total
466 ) {
467 // originally, auto-estimation would use the Root's timing primitives,
468 // but this makes it so that `state.estimatedTime` is in terms of the
469 // Root, which is unintuitive for everyone but the renderer.
470 const time = estimator.sample(Date.now(), state.value, state.total);
471 if (time && (!state.estimatedTime || state.estimatedTime !== time)) {
472 state.estimatedTime = time;
473 owner.emit("node-change", state, "estimatedTime");
474 }
475 }
476 const { parent } = state;
477 parent?.sortChildren && parent.children.sort(parent.sortChildren);
478 owner.emit("node-change", state, key);
479 }
480
481 const scope = log.headlessScope((m) => {
482 if (state.detached) return owner.emit("node-detached-log", m);
483 state.logs.push(m);
484 owner.emit("node-change", state, "logs");
485 });
486
487 const binding: Node = {
488 start(text, opts) {
489 if (state.detached) return nullNode;
490 const [child, node] = newNode(owner, text, opts);
491 state.children.push(child);
492 child.parent = state;
493 state.sortChildren && state.children.sort(state.sortChildren);
494 owner.emit("node-start", child, state);
495 owner.emit("node-change", state, "children");
496 return node;
497 },
498 get text() {
499 return state.text;
500 },
501 set text(value) {
502 state.text = value;
503 mutate("text");
504 },
505 get value() {
506 return state.value;
507 },
508 set value(value) {
509 state.value = value;
510 mutate("value");
511 },
512 get total() {
513 return state.total;
514 },
515 set total(value) {
516 state.total = value;
517 mutate("total");
518 },
519 get showTotal() {
520 return state.showTotal;
521 },
522 set showTotal(value) {
523 state.showTotal = value;
524 mutate("showTotal");
525 },
526 get passive() {
527 return state.passive;
528 },
529 set passive(value) {
530 state.passive = value;
531 mutate("passive");
532 },
533 get hidden() {
534 return state.hidden;
535 },
536 set hidden(value) {
537 state.hidden = value;
538 mutate("hidden");
539 },
540 get sortChildren() {
541 return state.sortChildren;
542 },
543 set sortChildren(value) {
544 state.sortChildren = value;
545 mutate("sortChildren");
546 },
547 get estimatedTime() {
548 return state.estimatedTime;
549 },
550 set estimatedTime(date) {
551 if (!estimator) {
552 state.estimatedTime = date;
553 mutate("estimatedTime");
554 }
555 },
556 get valueFormatter() {
557 return state.valueFormatter;
558 },
559 set valueFormatter(value) {
560 state.valueFormatter = value;
561 mutate("valueFormatter");
562 },
563 inc(delta = 1) {
564 state.value += delta;
565 if (state.total > 0 && state.value >= state.total) {
566 endNode(owner, state);
567 } else {
568 mutate("value");
569 }
570 },
571 log: scope,
572 end: () => void endNode(owner, state),
573 [Symbol.dispose]: () => void endNode(owner, state),
574 };
575 return [state, binding];
576}
577
578const noop = () => {};
579noop[Symbol.dispose] = noop;
580
581/** minimal implementation of {@linkcode Node} that has no I/O */
582export const nullNode: Node = {
583 start: () => nullNode,
584 get text() {
585 return "[detached]";
586 },
587 set text(_) {},
588 get value() {
589 return 0;
590 },
591 set value(_) {},
592 get total() {
593 return 0;
594 },
595 set total(_) {},
596 inc: noop,
597 log: {
598 info: noop,
599 warn: noop,
600 error: noop,
601 log: noop,
602 debug: noop,
603 writeMessage: noop,
604 write: noop,
605 scoped() {
606 return this;
607 },
608 tee: () => noop,
609 },
610 end: () => {},
611 [Symbol.dispose]: () => {},
612};
613
614function endNode<R, M extends Events.Map>(owner: Root<R, M>, state: Internal) {
615 if (state.detached) return;
616 state.children.forEach((child) => endNode(owner, child));
617 state.detached = true;
618 globalKeyPool.recycle(state.key);
619 if (state.parent) {
620 const i = state.parent.children.indexOf(state);
621 ASSERT(i !== -1);
622 state.parent.children.splice(i, 1);
623 }
624 owner.emit("node-end", state);
625 state.logs = [];
626 state.parent = null;
627}
628
629/** Convert the top level {@linkcode ReadOnlyNode|ReadOnlyNode[]} into ANSI text. */
630export function formatAnsi(now: number, list: readonly ReadOnlyNode[]): string {
631 let out = "";
632 for (const top of list) {
633 if (!isVisible(top)) continue;
634 out += renderAnsiMainLine(top, now, 0) + "\n";
635 out += renderChildren(top, now, []);
636 }
637 return out.trimEnd();
638}
639
640const spinnerFps = 12.5;
641const box = { tee: "├─ ", line: "│ ", langle: "└─ " };
642const barChars = [" ", "▏", "▎", "▍", "▌", "▋", "▊", "▉"];
643const fullBar = "█";
644const spinner = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"].map((frame) =>
645 ansi.style(ansi.fgBlue, frame),
646);
647
648function renderAnsiMainLine(state: ReadOnlyNode, now: number, depth: number) {
649 const { text, total, showTotal, value } = state;
650 const remaining = remainingSeconds(state);
651 const estimate = remaining ? ", " + string.formatDurationLetters(remaining) : "";
652 const valueFormatted = state.valueFormatter(value, total);
653 if (total && showTotal) {
654 return (
655 ansi.style(
656 ansi.bgBrightBlack + ansi.fgBlue,
657 formatUnicodeBar(value / total, Math.max(4, depth > 0 ? 12 : 25)),
658 ) +
659 (valueFormatted || estimate ? ` [${valueFormatted}${estimate}] ` : " ") +
660 text
661 );
662 }
663 const frame = Math.floor(now / (1000 / spinnerFps)) % spinner.length;
664 return (
665 (depth === 0 ? spinner[frame] + " " : "") +
666 (valueFormatted ? `[${valueFormatted}] ` : "") +
667 text
668 );
669}
670
671function renderChildren(state: ReadOnlyNode, now: number, depth: boolean[]) {
672 let maxHeight = 50; // TODO: flexible layout
673 let truncated = 0;
674 let out = "";
675 const children = state.children.filter(isVisible);
676 if (children.length === 0) return "";
677 for (let i = 0, { length } = children; i < length; i += 1) {
678 const child = UNWRAP(children[i]);
679 let item = "";
680 const left = depth.map((x) => (x ? box.line : " ")).join("");
681 item += left + (i === length - 1 && !truncated ? box.langle : box.tee);
682 item += renderAnsiMainLine(child, now, 1 + depth.length) + "\n";
683 item += renderChildren(child, now, depth.concat(i < length - 1));
684 const h = string.countNewlines(item);
685 if (h > maxHeight) {
686 truncated += 1;
687 continue;
688 }
689 const logLines = child.logs
690 .map((msg) => log.formatAnsiMessage(msg, true))
691 .join("")
692 .trim();
693 if (logLines) {
694 for (const line of logLines.split("\n").slice(-Math.min(maxLogLines, maxHeight))) {
695 item +=
696 left +
697 (i === length - 1 && !truncated ? " " : box.line) +
698 " " +
699 ansi.style(ansi.fgBrightBlack, ">") +
700 " " +
701 line +
702 "\n";
703 }
704 }
705 maxHeight -= h + string.countNewlines(logLines);
706 out += item;
707 }
708 if (truncated) {
709 out += depth.map((x) => (x ? box.line : " ")).join("") + box.langle;
710 out += ansi.style(ansi.fgBrightBlack, `[${truncated} more]`) + "\n";
711 }
712 return out;
713}
714
715/**
716 * This function is derived from an old program I wrote back in 2020 called `f`
717 * which did ffmpeg handling. It is probably one of the coolest progress bars
718 * ever imagined.
719 */
720export function formatUnicodeBar(progress: number, width: number): string {
721 if (progress >= 1) return fullBar.repeat(width);
722 if (progress <= 0 || Number.isNaN(progress)) return " ".repeat(width);
723
724 const wholeWidth = Math.floor(progress * width);
725 const remainderWidth = (progress * width) % 1;
726 const partWidth = Math.floor(remainderWidth * 8);
727 let partChar = barChars[partWidth];
728 if (width - wholeWidth - 1 < 0) partChar = "";
729
730 const fill = fullBar.repeat(wholeWidth);
731 const empty = " ".repeat(width - wholeWidth - 1);
732
733 return fill + partChar + empty;
734}
735
736/**
737 * configure a {@linkcode Root} to display its contents to a
738 * {@linkcode log.HeadlessWidgetHost}. this is used by the global progress
739 * instance to output to the terminal.
740 * ```ts
741 * const globalProgress = new progress.Root();
742 * progress.attachToScreen(log);
743 * ```
744 * unit tests use a mock screen instead of the terminal.
745 * ```ts
746 * const screen = new TestWidgetHost();
747 * const root = new progress.Root(screen.delay);
748 * root.attachToScreen(screen);
749 * // can use the progress API
750 * root.start("hello");
751 * // and then examine the contents
752 * screen.expectFrame(...);
753 * ```
754 */
755export function attachToScreen(
756 root: Root,
757 { writeOutput, startWidget }: Pick<log.WidgetHost, "writeOutput" | "startWidget">,
758): ts.Dispose {
759 const stack = new DisposableStack();
760 let widget: log.WidgetInstance | null = null;
761
762 stack.use(
763 root.on("change", (items) => {
764 if (items.length > 0) {
765 widget ??=
766 startWidget({
767 format: ({ now }) => formatAnsi(now, root.active),
768 }) ?? null;
769 if (!widget) return;
770 // scan for spinners and visible estimations
771 widget.fps = items.some((x) => !x.hidden && !(x.showTotal !== false && x.total > 0))
772 ? spinnerFps
773 : hasEstimate(items)
774 ? 1
775 : null;
776 widget.redraw();
777 } else {
778 widget?.stop();
779 widget = null;
780 }
781 }),
782 );
783 stack.use(
784 root.on("node-detached-log", (msg) => {
785 writeOutput(log.formatAnsiMessage(msg, true));
786 }),
787 );
788 stack.use(
789 root.on("node-end", (node) => {
790 let title = node.text;
791 let p: ReadOnlyNode | null = node;
792 while ((p = p.parent)) title = p.text + " / " + title;
793 const { logs } = node;
794 if (logs.length > 0) {
795 const header = `[logs from ${title}]`;
796 writeOutput(ansi.style(ansi.fgBrightBlack, header) + "\n");
797 writeOutput(logs.map((msg) => log.formatAnsiMessage(msg, true)).join(""));
798 }
799 }),
800 );
801
802 stack.defer(() => {
803 widget?.stop();
804 widget = null;
805 });
806
807 return ts.defer(() => stack.dispose());
808}
809
810function hasEstimate(nodes: readonly ReadOnlyNode[]): boolean {
811 return nodes.some((n) => n.estimatedTime != null || hasEstimate(n.children));
812}
813
814const header = /* @__PURE__ */ string.encodeUtf8("clover's progress <3\n");
815
816/** the 'Content-Type' of {@linkcode encodeByteStream}'s return value */
817export const contentType = "application/x-clover-progress";
818
819/**
820 * opaque but JSON-compatible serialized state, either containing the entire
821 * progress tree or just a delta update. this data structure always be future
822 * and backwards compatbile with changes to this library. if a breaking change
823 * is made, the APIs in this file will be renamed.
824 *
825 * for details on the format, read the source of {@linkcode encodeEventStream} or
826 * {@linkcode encodeByteStream}
827 */
828export type StreamEvent = Array<number | StreamNode | StreamCustomEvent | StreamRootChildren>;
829const kNode = Symbol("underlyingNode");
830/** @internal */
831export interface StreamNode {
832 k: EncodedKey;
833 /** text, overwrite */
834 t?: string | undefined;
835 /** value, overwrite */
836 v?: number | undefined;
837 /** total, overwrite */
838 e?: number | undefined;
839 /** showTotal, overwrite */
840 s?: boolean | undefined;
841 /** passive, overwrite */
842 p?: boolean | undefined;
843 /** logs, append */
844 l?: log.Message[] | undefined;
845 /** hidden, overwrite */
846 h?: boolean | undefined;
847 /** children, overwrite */
848 c?: EncodedKey[] | undefined;
849 /** estimated time, in unix milliseconds, overwrite */
850 E?: number | null | undefined;
851 /** formatted value if not defaultValueFormatter, overwrite */
852 V?: string | undefined;
853
854 /** access to the underlying node is used by {@linkcode encodeByteStream} */
855 [kNode]?: ReadOnlyNode;
856}
857type StreamCustomEvent = [
858 /** custom event name or "end" */
859 e: "end" | string,
860 /** custom event payload */
861 ...args: ts.Json[],
862];
863type StreamRootChildren = number[];
864
865/** options for both {@linkcode encodeEventStream} and {@linkcode encodeByteStream} */
866export interface EncodeStreamOptions {
867 /**
868 * set the minimum time between packets.
869 * @default 1000 / 30 (30fps)
870 */
871 throttleMs?: number;
872 /**
873 * `log.ts` is capable of capturing stack traces from all log calls. by
874 * seeing this to true, those traces will be serialized. disabled by default
875 * for privacy reasons.
876 */
877 serializeStackTraces?: boolean;
878}
879
880/**
881 * converts a {@linkcode Root|progress.Root} into an object stream
882 * for communicating progress over the process or network boundary.
883 *
884 * if the given root adds custom event handlers, they must all have
885 * json-serializable payloads.
886 */
887export function encodeEventStream<Result extends ts.Json, Map extends { [key: string]: ts.Json[] }>(
888 // fun fact, this type is impossible to write without generics
889 root: Root<Result, Map>,
890 { throttleMs = 1000 / 30 /* 30fps */, serializeStackTraces = false }: EncodeStreamOptions = {},
891): ReadableStream<StreamEvent> {
892 const { delay, now } = root;
893 const stack = new DisposableStack();
894 const s = new Encoder(root, serializeStackTraces);
895 const ready = new async.Watch(false);
896
897 let lastEvent = now();
898 let timer: async.Cancelable<void> | null = null;
899 function emitSoon() {
900 if (timer || ready.value) return;
901 const remaining = lastEvent + throttleMs - now();
902 if (remaining > 0) {
903 (timer = delay(remaining)).then(() => {
904 timer = null;
905 ready.value = true;
906 lastEvent = now();
907 });
908 } else {
909 ready.value = true;
910 lastEvent = now();
911 }
912 }
913 /** terminal events skip the throttle so the stream ends promptly */
914 function emitNow() {
915 timer?.cancel();
916 timer = null;
917 ready.value = true;
918 lastEvent = now();
919 }
920 stack.defer(() => {
921 if (timer) timer.cancel();
922 s.changed.clear();
923 s.deleted.clear();
924 });
925
926 let isFirst = true;
927 return new ReadableStream(
928 {
929 start(controller) {
930 if (root.active.length > 0) {
931 controller.enqueue(s.getDelta(isFirst));
932 isFirst = false;
933 }
934
935 stack.use(s.observe(emitSoon, emitNow));
936 },
937 async pull(controller) {
938 await ready.until((x) => x === true);
939 ready.value = false;
940 if (timer) {
941 timer.cancel();
942 timer = null;
943 }
944 controller.enqueue(s.getDelta(isFirst));
945 isFirst = false;
946 if (s.ended) {
947 controller.close();
948 stack.dispose();
949 }
950 },
951 cancel() {
952 stack.dispose();
953 },
954 },
955 { highWaterMark: 0 },
956 );
957}
958
959/**
960 * decodes a stream created by {@linkcode encodeEventStream} into managed calls
961 * to `target.start`. when cancelling, all the managed nodes are destroyed.
962 */
963export function decodeEventStream<Result = void, EventMap extends Events.Map = ts.EmptyObject>(
964 encoded: ReadableStream<StreamEvent>,
965 target: Ref,
966): async.Cancelable<Result> & Events<EventMap> {
967 const { resolve, reject, promise } = Promise.withResolvers<Result>();
968 const events = new Events<EventMap>();
969 const ctrl = new AbortController();
970 const decoder = new Decoder(target, events, resolve);
971 const { signal } = ctrl;
972 (async () => {
973 if (signal.aborted) return;
974 const reader = encoded.getReader();
975 const cancel = () => void reader.cancel(signal.reason).catch(() => {});
976 signal.addEventListener("abort", cancel, { once: true });
977 try {
978 let hasEmittedEnd = false;
979 while (true) {
980 signal.throwIfAborted();
981 const { value, done } = await reader.read();
982 signal.throwIfAborted();
983 if (done) break;
984 if (decoder.processEvent(value)) hasEmittedEnd = true;
985 }
986 ASSERT(hasEmittedEnd, "Stream terminated early.");
987 } catch (err) {
988 reader.cancel(err);
989 if (!signal.aborted) reject(err);
990 } finally {
991 signal.removeEventListener("abort", cancel);
992 decoder.reset();
993 reader.releaseLock();
994 }
995 })();
996 return Object.assign(ts.mixin(promise, events), {
997 cancel: (e?: unknown) => {
998 ctrl.abort(e);
999 decoder.reset();
1000 },
1001 [Symbol.dispose]: () => {
1002 ctrl.abort();
1003 decoder.reset();
1004 },
1005 });
1006}
1007
1008/**
1009 * by listening to the event interface in {@linkcode Root}, compute delta
1010 * events for a progress stream. the caller is expected to fill `changed` and `deleted`
1011 */
1012class Encoder<Result extends ts.Json, EventMap extends { [key: string]: ts.Json[] }> {
1013 root: Root<Result, EventMap>;
1014 pool: KeyPool<EncodedKey> = new KeyPool();
1015 keyMap: Map<ReadOnlyNode, EncodedKey> = new Map();
1016 logsLength: WeakMap<ReadOnlyNode, number> = new WeakMap();
1017 serializeStackTraces: boolean;
1018
1019 rootChildrenUpdated = true;
1020 changed = new Map<ReadOnlyNode, Set<keyof ReadOnlyNode>>();
1021 deleted = new Set<ReadOnlyNode>();
1022 /** custom and terminal ("end"/"error") events awaiting the next delta */
1023 pendingEvents: StreamCustomEvent[] = [];
1024 /** set when "end" or "error" was recorded; the stream closes after flushing */
1025 ended = false;
1026
1027 constructor(root: Root<Result, EventMap>, serializeStackTraces: boolean = false) {
1028 this.root = root;
1029 this.serializeStackTraces = serializeStackTraces;
1030 for (const node of root.active) {
1031 this.changed.set(node, new Set());
1032 }
1033 }
1034
1035 /**
1036 * fill `changed`, `deleted`, and `pendingEvents` from the events of the
1037 * root. `soon` is called when there is a delta worth sending, and `now`
1038 * when it is the last one.
1039 */
1040 observe(soon: () => void, now: () => void): DisposableStack {
1041 const { root } = this;
1042 const stack = new DisposableStack();
1043 // TODO: why does ts require two args
1044 stack.use(
1045 root.on("node-start", (newNode, _) => {
1046 if (!newNode.parent) this.rootChildrenUpdated = true;
1047 this.changed.set(newNode, new Set());
1048 soon();
1049 }),
1050 );
1051 stack.use(
1052 root.on("node-change", (node, key) => {
1053 let set = this.changed.get(node);
1054 if (!set) this.changed.set(node, (set = new Set()));
1055 set.add(key);
1056 soon();
1057 }),
1058 );
1059 stack.use(
1060 root.on("node-end", (node) => {
1061 if (!node.parent) this.rootChildrenUpdated = true;
1062 this.changed.delete(node);
1063 this.deleted.add(node); // handle add and remove in same frame
1064 soon();
1065 }),
1066 );
1067 // custom events pass through the stream verbatim
1068 stack.use(
1069 root.onAny((channel, args) => {
1070 if (rootEvents.has(channel)) return;
1071 this.pendingEvents.push([channel as string, ...(args as ts.Json[])]);
1072 soon();
1073 }),
1074 );
1075 stack.use(
1076 root.on("end", (result) => {
1077 this.pendingEvents.push(["end", result]);
1078 this.ended = true;
1079 now();
1080 }),
1081 );
1082 stack.use(
1083 root.on("error", (error) => {
1084 this.pendingEvents.push(["error", exceptions.message(error)]);
1085 this.ended = true;
1086 now();
1087 }),
1088 );
1089 return stack;
1090 }
1091
1092 getDelta(isFirst = false) {
1093 const deleted = Array.from(this.deleted, (node) => [node, this.keyMap.get(node)] as const)
1094 .filter(([node, dst]) => {
1095 if (dst == null) return false; // started and stopped in same tick
1096 this.pool.recycle(dst);
1097 this.keyMap.delete(node);
1098 return true;
1099 })
1100 .map((x) => UNWRAP(x[1]));
1101 ASSERT(!isFirst || deleted.length === 0);
1102
1103 const { updated = [], started = [] } = Object.groupBy(this.changed, ([node]) =>
1104 this.keyMap.has(node) ? "updated" : "started",
1105 );
1106
1107 const payload: StreamEvent = [isFirst ? -1 : deleted.length, ...deleted];
1108
1109 if (started.length > 0) {
1110 this.beginNodesDepthFirst(
1111 started.reverse().map(([node]) => node),
1112 payload,
1113 );
1114 }
1115
1116 for (const [node, props] of updated) {
1117 let logs = undefined;
1118 if (props.has("logs") && node.logs.length > (this.logsLength.get(node) ?? 0)) {
1119 logs = this.redactLogsIfNeeded(node.logs.slice(this.logsLength.get(node) ?? 0));
1120 this.logsLength.set(node, node.logs.length);
1121 }
1122 payload.push({
1123 k: UNWRAP(this.keyMap.get(node)),
1124 t: props.has("text") ? node.text : undefined,
1125 v: props.has("value") ? node.value : undefined,
1126 e: props.has("total") ? node.total : undefined,
1127 s: props.has("showTotal") ? node.showTotal : undefined,
1128 p: props.has("passive") ? node.passive : undefined,
1129 l: logs,
1130 h: props.has("hidden") ? node.hidden : undefined,
1131 c: props.has("children")
1132 ? node.children.map((node) => UNWRAP(this.keyMap.get(node)))
1133 : undefined,
1134 E: props.has("estimatedTime") ? node.estimatedTime : undefined,
1135 V:
1136 node.valueFormatter !== defaultValueFormatter &&
1137 (props.has("value") || props.has("total") || props.has("valueFormatter"))
1138 ? node.valueFormatter(node.value, node.total)
1139 : undefined,
1140 [kNode]: node,
1141 });
1142 }
1143
1144 if (this.rootChildrenUpdated) {
1145 payload.push(this.root.active.map((node) => UNWRAP(this.keyMap.get(node))));
1146 }
1147
1148 payload.push(...this.pendingEvents);
1149
1150 this.deleted.clear();
1151 this.changed.clear();
1152 this.pendingEvents = [];
1153 this.rootChildrenUpdated = false;
1154
1155 return payload;
1156 }
1157
1158 beginNodesDepthFirst(nodes: ReadOnlyNode[], out: StreamEvent) {
1159 for (let i = 0; i < nodes.length; i += 1) {
1160 const node = UNWRAP(nodes[i]);
1161 nodes.push(...node.children);
1162 }
1163 let next;
1164 while ((next = nodes.pop())) {
1165 if (!this.keyMap.has(next)) {
1166 out.push(this.beginNode(next));
1167 }
1168 }
1169 }
1170
1171 beginNode(node: ReadOnlyNode): StreamNode {
1172 const k = this.pool.get();
1173 this.keyMap.set(node, k);
1174
1175 return {
1176 k,
1177 v: node.value !== 0 ? node.value : undefined,
1178 e: node.total !== 0 ? node.total : undefined,
1179 t: node.text,
1180 l:
1181 node.logs.length > 0
1182 ? (this.logsLength.set(node, node.logs.length),
1183 this.redactLogsIfNeeded(node.logs.slice()))
1184 : undefined,
1185 c:
1186 node.children.length > 0
1187 ? node.children.map((node) =>
1188 UNWRAP(this.keyMap.get(node), () => [node.children, this.keyMap]),
1189 )
1190 : undefined,
1191 p: node.passive === true ? true : undefined,
1192 s: node.showTotal === false ? false : undefined,
1193 h: node.hidden === true ? true : undefined,
1194 E: node.estimatedTime != null ? node.estimatedTime : undefined,
1195 V:
1196 node.valueFormatter !== defaultValueFormatter
1197 ? node.valueFormatter(node.value, node.total)
1198 : undefined,
1199 [kNode]: node,
1200 };
1201 }
1202
1203 redactLogsIfNeeded(messages: log.Message[]) {
1204 if (this.serializeStackTraces) return messages;
1205 return messages.map((x) => ({ ...x, stack: undefined }));
1206 }
1207}
1208
1209class Decoder<Result = void, EventMap extends Events.Map = ts.EmptyObject> {
1210 target: Ref;
1211 events: Events<EventMap>;
1212 resolve: (result: Result) => void;
1213 active = new Map<EncodedKey, Node>();
1214 pendingStart = new Map<EncodedKey, PendingStart>();
1215 pendingParents = new Map<EncodedKey, EncodedKey>();
1216
1217 constructor(target: Ref, events: Events<EventMap>, resolve: (result: Result) => void) {
1218 this.target = target;
1219 this.events = events;
1220 this.resolve = resolve;
1221 }
1222
1223 reset() {
1224 for (const v of this.active.values()) v.end();
1225 this.active.clear();
1226 }
1227
1228 /** returns whether the stream signalled "end"; throws on "error" */
1229 processEvent(event: StreamEvent): boolean {
1230 event = event.slice();
1231
1232 // delete unreferenced nodes first, since they may get re-allocated
1233 // in the next phase.
1234 const deletedCount = event.shift();
1235 ASSERT(typeof deletedCount === "number");
1236 if (deletedCount === -1) {
1237 // connection reset, delete all nodes
1238 this.reset();
1239 } else {
1240 for (const id of event.splice(0, deletedCount)) {
1241 ASSERT(typeof id === "number");
1242 UNWRAP(
1243 this.active.get(id as EncodedKey),
1244 () => `dstkey ${id} not found, ${[...this.active.keys()]}`,
1245 ).end();
1246 ASSERT(this.active.delete(id as EncodedKey));
1247 }
1248 }
1249
1250 // parse all of the events, updating all nodes and extracting creations
1251 let resolving = false;
1252 let resolvingValue: unknown = null;
1253 let updateChildren: Array<{
1254 key: EncodedKey;
1255 children: EncodedKey[];
1256 }> = [];
1257 for (const chunk of event) {
1258 ASSERT(typeof chunk === "object");
1259 if (Array.isArray(chunk)) {
1260 if (typeof chunk[0] === "string") {
1261 if (chunk[0] === "end") {
1262 resolving = true;
1263 resolvingValue = chunk[1];
1264 } else if (chunk[0] === "error") {
1265 throw new Error(`Progress stream error: ${chunk[1]}`);
1266 } else {
1267 // custom event
1268 // @ts-expect-error TODO: typescript soundness
1269 this.events.emit(...chunk);
1270 }
1271 } else {
1272 const ids = chunk as EncodedKey[];
1273 for (const id of ids) this.pendingParents.set(id, 0 as EncodedKey);
1274 }
1275 } else {
1276 // node update or creation
1277 const {
1278 k: key,
1279 t: text,
1280 v: value,
1281 e: total,
1282 s: showTotal,
1283 p: passive,
1284 h: hidden,
1285 l: messages,
1286 c: children,
1287 E: estimatedTime,
1288 V: formattedValue,
1289 } = chunk;
1290 const node = this.active.get(key);
1291 if (children) {
1292 for (const id of children) this.pendingParents.set(id, key);
1293 updateChildren.push({ key, children });
1294 }
1295 if (node) {
1296 if (text !== undefined) node.text = text;
1297 if (value !== undefined) node.value = value;
1298 if (total !== undefined) node.total = total;
1299 if (showTotal != null) node.showTotal = showTotal;
1300 if (passive != null) node.passive = passive;
1301 if (hidden != null) node.hidden = hidden;
1302 for (const message of messages ?? []) node.log.writeMessage(message);
1303 if (estimatedTime !== undefined) node.estimatedTime = estimatedTime;
1304 if (formattedValue !== undefined) node.valueFormatter = () => formattedValue;
1305 for (const id of children ?? []) this.pendingParents.set(id, key);
1306 } else {
1307 ASSERT(text != null, `DstKey ${key} not active but not created`);
1308 this.pendingStart.set(key, {
1309 text,
1310 value,
1311 total,
1312 showTotal,
1313 passive,
1314 hidden,
1315 messages,
1316 estimatedTime,
1317 formattedValue,
1318 });
1319 }
1320 }
1321 }
1322
1323 let entry;
1324 while ((entry = this.pendingStart.entries().next().value)) {
1325 const [key, value] = entry;
1326 this.startRecursive(key, value);
1327 }
1328 this.pendingParents.clear();
1329
1330 // validate that every referenced child became active; a failure here
1331 // means the stream is desynced
1332 for (const { key, children } of updateChildren) {
1333 UNWRAP(this.active.get(key), () => `parent ${key} not active`);
1334 for (const id of children) {
1335 UNWRAP(this.active.get(id), () => `child ${id} not active`);
1336 }
1337 }
1338
1339 if (resolving) {
1340 this.reset();
1341 this.resolve(resolvingValue as Result);
1342 return true;
1343 }
1344 return false;
1345 }
1346
1347 startRecursive(key: EncodedKey, opts: PendingStart): Ref {
1348 ASSERT(this.pendingStart.delete(key));
1349 // a node whose parent linkage is missing (malformed stream) attaches at
1350 // the root rather than killing the whole decoder
1351 const parentId = this.pendingParents.get(key) ?? (0 as EncodedKey);
1352 let ref: Ref | null = parentId === 0 ? this.target : (this.active.get(parentId) ?? null);
1353 if (!ref) {
1354 ref = this.startRecursive(parentId, UNWRAP(this.pendingStart.get(parentId)));
1355 }
1356 const { text, estimatedTime, formattedValue, ...options } = opts;
1357 const node = ref.start(text, {
1358 ...options,
1359 estimateCompletion: false,
1360 });
1361 this.active.set(key, node);
1362 node.estimatedTime = estimatedTime;
1363 if (formattedValue !== undefined) node.valueFormatter = () => formattedValue;
1364 this.pendingParents.delete(key);
1365 return node;
1366 }
1367}
1368
1369interface PendingStart {
1370 text: string;
1371 value?: number | undefined;
1372 total?: number | undefined;
1373 showTotal?: boolean | undefined;
1374 passive?: boolean | undefined;
1375 hidden?: boolean | undefined;
1376 messages?: log.Message[] | undefined;
1377 estimatedTime?: number | null | undefined;
1378 formattedValue?: string | undefined;
1379}
1380
1381/** this interface must not have more than 16 properties */
1382interface DeltaFlags {
1383 // the new status is always present since it's one bit
1384 hidden: boolean;
1385 passive: boolean;
1386 showTotal: boolean;
1387
1388 // for each packet that changed it stores out of line
1389 changedValue: boolean;
1390 changedTotal: boolean;
1391 changedLogs: boolean;
1392 changedText: boolean;
1393 changedChildren: boolean;
1394 changedEstimatedTime: boolean;
1395 changedFormattedValue: boolean;
1396
1397 estimatedTimeNonNull: boolean;
1398
1399 futureProofingPacket: boolean;
1400 deleted: boolean;
1401}
1402
1403/**
1404 * NOTE: this function contains bugs and its format is not yet stabilized.
1405 *
1406 * converts a {@linkcode Root|progress.Root} into an byte stream for
1407 * communicating progress over the process or network boundary. the output is a
1408 * raw binary payload that uses an extremely compact representation for the
1409 * nodes. if trying to transmit UTF-8, consider the JSON-encoding
1410 * {@linkcode encodeEventStream}
1411 *
1412 * if the given root adds custom event handlers, they must all have
1413 * json-serializable payloads.
1414 */
1415export function encodeByteStream<
1416 Result extends ts.Json,
1417 EventMap extends { [key: string]: ts.Json[] },
1418>(root: Root<Result, EventMap>, options?: EncodeStreamOptions): ReadableStream<Uint8Array> {
1419 const events = encodeEventStream(root, options);
1420 const transform = new TransformStream({ transform: writeStreamEvent });
1421 return events.pipeThrough(transform);
1422}
1423
1424/** convert {@linkcode StreamEvent} into a byte payload. Mutates the input event. */
1425function writeStreamEvent(event: StreamEvent, writer: stream.WriteTarget) {
1426 using w = new stream.BufferedWriter(writer);
1427
1428 let deleteCount = event.shift();
1429 ASSERT(typeof deleteCount === "number");
1430
1431 if (deleteCount === -1) {
1432 deleteCount = 0;
1433 w.write(header);
1434 } else {
1435 w.u8("\n".charCodeAt(0));
1436
1437 const deletedNodes = event.splice(0, deleteCount);
1438 w.varUint(deletedNodes.length);
1439 for (const dstKey of deletedNodes) {
1440 ASSERT(typeof dstKey === "number");
1441 w.varUint(dstKey);
1442 }
1443 }
1444
1445 const rootChildren = event.find(
1446 (x): x is StreamRootChildren => Array.isArray(x) && typeof x[0] === "number",
1447 );
1448 const changedNodes = event.filter(
1449 (x): x is StreamNode => typeof x === "object" && !Array.isArray(x),
1450 );
1451 w.varUint(changedNodes.length + +!!rootChildren);
1452 for (const obj of changedNodes) {
1453 ASSERT(typeof obj === "object" && "k" in obj);
1454 const {
1455 k: key,
1456 t: text,
1457 v: value,
1458 e: total,
1459 l: messages,
1460 c: childrenSet,
1461 E: estimatedTime,
1462 V: formattedValue,
1463 } = obj;
1464 const { hidden, showTotal, passive } = UNWRAP(obj[kNode]);
1465 ASSERT(key > 0);
1466 ASSERT(messages == null || messages.length > 0);
1467
1468 w.varUint(key);
1469 w.u16(
1470 encodeDeltaFlags({
1471 hidden,
1472 passive,
1473 showTotal,
1474 changedValue: value !== undefined,
1475 changedTotal: total !== undefined,
1476 changedLogs: messages !== undefined,
1477 changedText: text !== undefined,
1478 changedChildren: childrenSet !== undefined,
1479 changedEstimatedTime: estimatedTime !== undefined,
1480 changedFormattedValue: formattedValue !== undefined,
1481 estimatedTimeNonNull: estimatedTime !== null,
1482 }),
1483 );
1484 if (value !== undefined) w.f64(value);
1485 if (total !== undefined) w.f64(total);
1486 if (text !== undefined) w.stringWithLength(text);
1487 if (messages !== undefined) {
1488 w.varUint(messages.length);
1489 for (const msg of messages) {
1490 let level = logLevelSerialize.indexOf(msg.level ?? "info");
1491 if (level === -1) level = 0;
1492 w.u8(
1493 level +
1494 (msg.newline === false ? 1 << 4 : 0) +
1495 (msg.scope ? 1 << 5 : 0) +
1496 (msg.stack && msg.stack.length > 0 ? 1 << 6 : 0) +
1497 (msg.custom !== undefined ? 1 << 7 : 0),
1498 );
1499 w.stringWithLength(msg.text);
1500 w.varUint(Math.floor(msg.time));
1501 if (msg.scope) w.stringWithLength(msg.scope);
1502 if (msg.stack && msg.stack.length > 0) {
1503 w.varUint(msg.stack.length);
1504 for (const { fn, file, line, col } of msg.stack) {
1505 w.stringWithLength(fn ?? "");
1506 w.stringWithLength(file ?? "");
1507 w.varUint(line ?? 0);
1508 w.varUint(col ?? 0);
1509 }
1510 }
1511 if (msg.custom !== undefined) {
1512 w.stringWithLength(JSON.stringify(msg.custom));
1513 }
1514 }
1515 }
1516 if (childrenSet !== undefined) {
1517 w.varUint(childrenSet.length);
1518 for (const key of childrenSet) w.varUint(key);
1519 }
1520 if (estimatedTime != null) w.varUint(Math.floor(estimatedTime));
1521 if (formattedValue !== undefined) {
1522 w.stringWithLength(formattedValue);
1523 }
1524 }
1525 if (rootChildren) {
1526 w.varUint(0);
1527 w.varUint(rootChildren.length);
1528 for (const key of rootChildren) {
1529 w.varUint(key);
1530 }
1531 }
1532
1533 const customEvents = event.filter(
1534 (x): x is StreamCustomEvent => Array.isArray(x) && typeof x[0] === "string",
1535 );
1536 w.varUint(customEvents.length);
1537 for (const [event, ...args] of customEvents) {
1538 w.stringWithLength(event);
1539 w.stringWithLength(JSON.stringify(args));
1540 }
1541}
1542
1543/**
1544 * decodes a stream created by {@linkcode encodeByteStream} into managed calls to
1545 * `target.start`. when cancelling, all the managed nodes are destroyed.
1546 */
1547export function decodeByteStream<Result = void, EventMap extends Events.Map = ts.EmptyObject>(
1548 encoded: ReadableStream<Uint8Array>,
1549 target: Ref,
1550): async.Cancelable<Result> & Events<EventMap> {
1551 let cancelled = false;
1552 let reader: stream.BufferedReader | null = null;
1553 return decodeEventStream<Result, EventMap>(
1554 new ReadableStream<StreamEvent>({
1555 async start(controller) {
1556 reader = new stream.BufferedReader(encoded.getReader());
1557 try {
1558 // an event boundary with no further bytes is a clean end of stream
1559 while (reader && (await reader.ensureAvailableOrFalse(1))) {
1560 controller.enqueue(await readStreamEvent(reader));
1561 }
1562 if (!cancelled) controller.close();
1563 } catch (e) {
1564 if (!cancelled) {
1565 reader?.cancel(e);
1566 throw e;
1567 }
1568 } finally {
1569 reader?.releaseLock();
1570 reader = null;
1571 }
1572 },
1573 cancel(reason) {
1574 cancelled = true;
1575 reader?.releaseLock();
1576 reader = null;
1577 encoded.cancel(reason);
1578 },
1579 }),
1580 target,
1581 );
1582}
1583
1584async function readStreamEvent(r: stream.BufferedReader): Promise<StreamEvent> {
1585 const event: StreamEvent = [];
1586 const start = await r.peekU8();
1587 if (start === "\n".charCodeAt(0)) {
1588 await r.u8();
1589 let deletedCount = await r.varUint();
1590 event.push(deletedCount);
1591 while (deletedCount-- > 0) event.push(await r.varUint());
1592 } else {
1593 const readHeader = await r.readExactly(header.byteLength);
1594 if (!bytes.eql(readHeader, header)) {
1595 const m = JSON.stringify(string.decodeUtf8(readHeader));
1596 throw new Error(`Header mismatch, got ${m}`);
1597 }
1598 event.push(-1);
1599 }
1600 const changedNodes = await r.varUint();
1601 for (let i = 0; i < changedNodes; i += 1) {
1602 const key = await r.varUint();
1603 if (key === 0) {
1604 let count = await r.varUint();
1605 const array: StreamRootChildren = [];
1606 while (count-- > 0) array.push(await r.varUint());
1607 event.push(array);
1608 continue;
1609 }
1610 const flags = decodeDeltaFlags(await r.u16());
1611 const v = flags.changedValue ? await r.f64() : undefined;
1612 const e = flags.changedTotal ? await r.f64() : undefined;
1613 const t = flags.changedText ? await r.stringWithLength() : undefined;
1614 let l: undefined | log.Message[];
1615 if (flags.changedLogs) {
1616 l = [];
1617 let len = await r.varUint();
1618 while (len-- > 0) {
1619 const msgFlags = await r.u8();
1620 const level = logLevelSerialize[msgFlags & 0b1111] ?? "info";
1621 // bitwise tests: `msgFlags && (1 << 5) > 0` parsed as
1622 // `msgFlags && true`, which fabricated scope/stack/custom reads for
1623 // any message with a non-zero level and desynced the entire stream
1624 const partial = (msgFlags & (1 << 4)) !== 0;
1625 const hasScope = (msgFlags & (1 << 5)) !== 0;
1626 const hasStack = (msgFlags & (1 << 6)) !== 0;
1627 const hasCustom = (msgFlags & (1 << 7)) !== 0;
1628 const text = await r.stringWithLength();
1629 const time = await r.varUint();
1630 const scope = hasScope ? await r.stringWithLength() : undefined;
1631 let stack: stack.Frame[] | undefined = undefined;
1632 if (hasStack) {
1633 stack = [];
1634 let frames = await r.varUint();
1635 while (frames-- > 0) {
1636 const fn = await r.stringWithLength();
1637 const file = await r.stringWithLength();
1638 const line = await r.varUint();
1639 const col = await r.varUint();
1640 stack.push({
1641 fn: fn || null,
1642 file: file || null,
1643 line: line || null,
1644 col: col || null,
1645 });
1646 }
1647 }
1648 const custom = hasCustom ? JSON.parse(await r.stringWithLength()) : undefined;
1649 l.push({ level, text, time, scope, stack, custom, newline: partial ? false : undefined });
1650 }
1651 }
1652 let c: undefined | EncodedKey[] = undefined;
1653 if (flags.changedChildren) {
1654 c = [];
1655 let len = await r.varUint();
1656 while (len-- > 0) c.push((await r.varUint()) as EncodedKey);
1657 }
1658 const E = flags.changedEstimatedTime
1659 ? flags.estimatedTimeNonNull
1660 ? await r.varUint()
1661 : null
1662 : undefined;
1663 const V = flags.changedFormattedValue ? await r.stringWithLength() : undefined;
1664
1665 // a "v2"'s packet will go here, contents ignored by this version.
1666 if (flags.futureProofingPacket) {
1667 const len = await r.varUint();
1668 void (await r.readExactly(len));
1669 }
1670
1671 event.push({
1672 k: key as EncodedKey,
1673 t,
1674 v,
1675 e,
1676 s: flags.showTotal,
1677 p: flags.passive,
1678 l,
1679 h: flags.hidden,
1680 c,
1681 E,
1682 V,
1683 });
1684 }
1685
1686 // events
1687 let eventCount = await r.varUint();
1688 while (eventCount-- > 0) {
1689 const name = await r.stringWithLength();
1690 const args = JSON.parse(await r.stringWithLength());
1691 event.push([name, ...args]);
1692 }
1693
1694 return event;
1695}
1696
1697const logLevelSerialize = ["info", "error", "warn", "debug"] as const;
1698
1699function encodeDeltaFlags(node: Partial<DeltaFlags>) {
1700 return (
1701 (node.hidden ? 1 << 0 : 0) +
1702 (node.passive ? 1 << 1 : 0) +
1703 (node.showTotal ? 1 << 2 : 0) +
1704 (node.changedValue ? 1 << 3 : 0) +
1705 (node.changedTotal ? 1 << 4 : 0) +
1706 (node.changedLogs ? 1 << 6 : 0) +
1707 (node.changedText ? 1 << 7 : 0) +
1708 (node.changedChildren ? 1 << 8 : 0) +
1709 (node.changedEstimatedTime ? 1 << 9 : 0) +
1710 (node.changedFormattedValue ? 1 << 10 : 0) +
1711 (node.estimatedTimeNonNull ? 1 << 11 : 0) +
1712 (node.futureProofingPacket ? 1 << 14 : 0) +
1713 (node.deleted ? 1 << 15 : 0)
1714 );
1715}
1716
1717function decodeDeltaFlags(flags: number): DeltaFlags {
1718 return {
1719 hidden: (flags & (1 << 0)) !== 0,
1720 passive: (flags & (1 << 1)) !== 0,
1721 showTotal: (flags & (1 << 2)) !== 0,
1722 changedValue: (flags & (1 << 3)) !== 0,
1723 changedTotal: (flags & (1 << 4)) !== 0,
1724 changedLogs: (flags & (1 << 6)) !== 0,
1725 changedText: (flags & (1 << 7)) !== 0,
1726 changedChildren: (flags & (1 << 8)) !== 0,
1727 changedEstimatedTime: (flags & (1 << 9)) !== 0,
1728 changedFormattedValue: (flags & (1 << 10)) !== 0,
1729 estimatedTimeNonNull: (flags & (1 << 11)) !== 0,
1730 futureProofingPacket: (flags & (1 << 14)) !== 0,
1731 deleted: (flags & (1 << 15)) !== 0,
1732 };
1733}
1734
1735class KeyPool<T extends number> {
1736 next = 0;
1737 old: T[] = [];
1738
1739 get() {
1740 const existing = this.old.shift();
1741 if (existing) return existing as T;
1742 return (this.next += 1) as T;
1743 }
1744
1745 recycle(k: T) {
1746 this.old.push(k);
1747 }
1748}
1749
1750/** stateful estimation by providing samples */
1751export interface EstimationAlgorithm {
1752 /**
1753 * given a timestamp `time` and a progress value `progress`, return the
1754 * timestamp that the action will be finished on. called with (t, 0, 0)
1755 * to initialize the algorithm.
1756 */
1757 sample(time: number, current: number, total: number): number | null;
1758}
1759
1760/**
1761 * Exponential Moving Average.
1762 * this is the algorithm used for the default progress estimator.
1763 * https://en.wikipedia.org/wiki/Exponential_smoothing
1764 */
1765export class Ema implements EstimationAlgorithm {
1766 /** milliseconds */
1767 start: number | null = null;
1768 estimate: number | null = null;
1769 /** smoothing factor (0-1) */
1770 alpha: number;
1771 samples = 0;
1772
1773 constructor(alpha: number = 0.25) {
1774 this.alpha = alpha;
1775 }
1776
1777 sample(time: number, current: number, total: number): number | null {
1778 this.samples += 1;
1779
1780 ASSERT(current >= 0 && current <= total);
1781 if (this.start == null) {
1782 this.start = time;
1783 this.estimate = null;
1784 return null;
1785 }
1786
1787 const elapsed = time - this.start;
1788 if (elapsed === 0) return null; // too quick
1789
1790 const rate = current / elapsed;
1791 this.estimate =
1792 this.estimate != null ? (1 - this.alpha) * this.estimate + this.alpha * rate : rate;
1793 if (elapsed < 10_000 || this.estimate <= 0 || this.samples < 3) return null;
1794 const remaining = total - current;
1795 return time + remaining / this.estimate;
1796 }
1797
1798 reset(): void {
1799 this.start = null;
1800 this.estimate = null;
1801 }
1802}
1803
1804/** according to a stream */
1805type EncodedKey = number & { brand: typeof kNode };
1806
1807const globalKeyPool = new KeyPool<number>();
1808
1809/** names the pipe to a parent process that handed this one a place in its progress */
1810const ipcEnvVar = "CLOVER_PROGRESS";
1811
1812/**
1813 * the environment for a child process whose global root and log are to be
1814 * read by {@linkcode decodeAttachToRef}. `fd` is where the child has the pipe.
1815 */
1816export function ipcChildEnvironment(fd: number): Record<string, string> {
1817 return { [ipcEnvVar]: `${fd}:${UNWRAP(node.process).pid}` };
1818}
1819
1820export async function decodeAttachToRef(
1821 readable: ReadableStream<Uint8Array>,
1822 target: Ref,
1823): Promise<void> {
1824 const decoder = new Decoder(target, new Events(), noop);
1825 try {
1826 for await (const line of stream.lines(readable)) {
1827 const record: StreamEvent | log.Message = JSON.parse(line);
1828 if (Array.isArray(record)) decoder.processEvent(record);
1829 else log.writeMessage(record);
1830 }
1831 } finally {
1832 decoder.reset();
1833 }
1834}
1835
1836/**
1837 * the other end of {@linkcode decodeAttachToRef}. writes are synchronous, so that
1838 * the last words of a process are not lost when it exits.
1839 */
1840function tryAttachToParent(root: Root): boolean {
1841 const { process } = node;
1842 const fs = node.builtin("fs");
1843 const [fd, parent] = (process?.env[ipcEnvVar] ?? "").split(":").map(Number);
1844 // every descendant inherits the variable, but only the child has the pipe
1845 if (!process || !fs || !fd || process.ppid !== parent) return false;
1846
1847 const write = (record: StreamEvent | log.Message) => {
1848 try {
1849 fs.writeSync(fd, JSON.stringify(record) + "\n");
1850 } catch {
1851 // the parent is gone, and so is anyone to tell
1852 }
1853 };
1854
1855 const encoder = new Encoder(root);
1856 let timer: async.Cancelable<void> | null = null;
1857 const flush = () => {
1858 timer?.cancel();
1859 timer = null;
1860 write(encoder.getDelta());
1861 };
1862 const flushSoon = () => {
1863 if (timer) return;
1864 (timer = root.delay(1000 / 30)).then(flush);
1865 };
1866
1867 encoder.observe(flushSoon, flush);
1868 process.addListener("exit", () => timer && flush());
1869 // what changed on a node before a message is said stays before it
1870 log.replaceGlobalMessageDestination((m) => (timer && flush(), write({ ...m, stack: undefined })));
1871 return true;
1872}
1873
1874const global: Ref = /** @__PURE__ */ ((root = new Root()) => (
1875 tryAttachToParent(root) || attachToScreen(root, log),
1876 root
1877))();
1878
1879/**
1880 * a {@linkcode Ref} to the global progress root. unlike referencing the
1881 * namespace import, this value is tree-shakable.
1882 */
1883export const globalRoot: Ref = { start: (text, opts) => global.start(text, opts) };
1884
1885/**
1886 * for testing. not covered by semver
1887 * @internal
1888 */
1889export const internals: {
1890 readStreamEvent: typeof readStreamEvent;
1891 writeStreamEvent: typeof writeStreamEvent;
1892 kNode: symbol;
1893 EncodedKey: EncodedKey;
1894} = /** @__PURE__ */ (() => ({
1895 readStreamEvent,
1896 writeStreamEvent,
1897 kNode: kNode as symbol,
1898 EncodedKey: 0 as EncodedKey,
1899}))();
1900
1901import { ASSERT, UNWRAP } from "./assert.ts";
1902import * as async from "./async.ts";
1903import * as bytes from "./bytes.ts";
1904import { Events } from "./Events.ts";
1905import * as exceptions from "./exception.ts";
1906import * as log from "./log.ts";
1907import * as stack from "./log/stack.ts";
1908import * as math from "./math.ts";
1909import * as node from "./node.ts";
1910import * as stream from "./stream.ts";
1911import * as string from "./string.ts";
1912import * as ansi from "./string/ansi.ts";
1913import * as ts from "./ts.ts";
1914
1915import { isVisible, maxLogLines, remainingSeconds } from "./progress-format.ts";
1916
1917export { isVisible };