1import * as Effect from "effect/Effect";
2import * as FileSystem from "effect/FileSystem";
3import * as Path from "effect/Path";
4import * as PlatformError from "effect/PlatformError";
5import * as Schema from "effect/Schema";
6
7/**
8 * Read-only decoder for a Chromium profile's `Local Storage/leveldb`.
9 *
10 * It reads the database files directly and never takes LevelDB's lock, so it
11 * works while another app version has the profile open. Live files come from
12 * the MANIFEST that CURRENT names, and the newest sequence number wins, which
13 * is how LevelDB itself resolves overwrites and deletions.
14 */
15
16const LOG_BLOCK_SIZE = 32 * 1024;
17const LOG_HEADER_SIZE = 7;
18const TABLE_FOOTER_SIZE = 48;
19const BLOCK_TRAILER_SIZE = 5;
20
21class LevelDbFormatError extends Schema.TaggedError<LevelDbFormatError>()("LevelDbFormatError", {
22 detail: Schema.String,
23}) {}
24
25const isLevelDbFormatError = Schema.is(LevelDbFormatError);
26
27const fail = (detail: string): never => {
28 throw new LevelDbFormatError({ detail });
29};
30
31// ignoreBOM keeps a leading U+FEFF that is part of the stored string.
32const textDecoderUtf16 = new TextDecoder("utf-16le", { ignoreBOM: true });
33const textDecoderUtf8 = new TextDecoder();
34
35/** True ISO-8859-1. WHATWG's "latin1" label is windows-1252, which remaps 0x80-0x9F. */
36function decodeLatin1(bytes: Uint8Array): string {
37 let result = "";
38 for (let offset = 0; offset < bytes.length; offset += 0x8000) {
39 result += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000));
40 }
41 return result;
42}
43
44class Reader {
45 readonly bytes: Uint8Array;
46 offset: number;
47 constructor(bytes: Uint8Array, offset = 0) {
48 this.bytes = bytes;
49 this.offset = offset;
50 }
51 get done() {
52 return this.offset >= this.bytes.length;
53 }
54 byte(): number {
55 if (this.offset >= this.bytes.length) fail("unexpected end of data");
56 return this.bytes[this.offset++]!;
57 }
58 varint(): number {
59 let result = 0;
60 let multiplier = 1;
61 for (let shift = 0; shift < 64; shift += 7) {
62 const byte = this.byte();
63 result += (byte & 0x7f) * multiplier;
64 if ((byte & 0x80) === 0) return result;
65 multiplier *= 128;
66 }
67 return fail("varint too long");
68 }
69 take(length: number): Uint8Array {
70 if (length < 0 || this.offset + length > this.bytes.length) fail("length out of range");
71 const slice = this.bytes.subarray(this.offset, this.offset + length);
72 this.offset += length;
73 return slice;
74 }
75 lengthPrefixed(): Uint8Array {
76 return this.take(this.varint());
77 }
78}
79
80const readUint32 = (bytes: Uint8Array, offset: number) =>
81 (bytes[offset]! | (bytes[offset + 1]! << 8) | (bytes[offset + 2]! << 16)) +
82 bytes[offset + 3]! * 0x1000000;
83
84const CRC32C_TABLE = (() => {
85 const table = new Uint32Array(256);
86 for (let index = 0; index < 256; index++) {
87 let crc = index;
88 for (let bit = 0; bit < 8; bit++) crc = crc & 1 ? (crc >>> 1) ^ 0x82f63b78 : crc >>> 1;
89 table[index] = crc >>> 0;
90 }
91 return table;
92})();
93
94function crc32c(bytes: Uint8Array): number {
95 let crc = 0xffffffff;
96 for (const byte of bytes) crc = CRC32C_TABLE[(crc ^ byte) & 0xff]! ^ (crc >>> 8);
97 return (crc ^ 0xffffffff) >>> 0;
98}
99
100/** LevelDB stores CRCs masked so a CRC of data containing CRCs stays well distributed. */
101const unmaskCrc = (masked: number) => {
102 const rotated = (masked - 0xa282ead8) >>> 0;
103 return ((rotated >>> 17) | (rotated << 15)) >>> 0;
104};
105
106function decompressSnappy(input: Uint8Array): Uint8Array {
107 const reader = new Reader(input);
108 const output = new Uint8Array(reader.varint());
109 let written = 0;
110 while (!reader.done) {
111 const tag = reader.byte();
112 const kind = tag & 3;
113 if (kind === 0) {
114 let length = tag >>> 2;
115 if (length >= 60) {
116 const extraBytes = length - 59;
117 length = 0;
118 for (let index = 0; index < extraBytes; index++) length += reader.byte() * 256 ** index;
119 }
120 length += 1;
121 if (written + length > output.length) fail("snappy literal overflows output");
122 output.set(reader.take(length), written);
123 written += length;
124 continue;
125 }
126 let length: number;
127 let offset: number;
128 if (kind === 1) {
129 length = 4 + ((tag >>> 2) & 7);
130 offset = ((tag >>> 5) << 8) | reader.byte();
131 } else if (kind === 2) {
132 length = (tag >>> 2) + 1;
133 offset = reader.byte() | (reader.byte() << 8);
134 } else {
135 length = (tag >>> 2) + 1;
136 offset = readUint32(reader.take(4), 0);
137 }
138 if (offset === 0 || offset > written || written + length > output.length) {
139 fail("snappy copy out of range");
140 }
141 // Copies may overlap their own output, so they go byte by byte.
142 for (let index = 0; index < length; index++) {
143 output[written] = output[written - offset]!;
144 written++;
145 }
146 }
147 if (written !== output.length) fail("snappy output length mismatch");
148 return output;
149}
150
151/**
152 * Splits a LevelDB log (write-ahead log or MANIFEST) into its records. Like
153 * LevelDB's own reader, a corrupt record or zero padding skips the rest of its
154 * block, since a reused log keeps appending after a torn write, and a record
155 * cut off by the end of the file ends the log.
156 */
157function readLogRecords(bytes: Uint8Array): Uint8Array[] {
158 const records: Uint8Array[] = [];
159 let pending: Uint8Array[] | null = null;
160 let offset = 0;
161 while (offset + LOG_HEADER_SIZE <= bytes.length) {
162 const blockRemaining = LOG_BLOCK_SIZE - (offset % LOG_BLOCK_SIZE);
163 if (blockRemaining < LOG_HEADER_SIZE) {
164 offset += blockRemaining;
165 continue;
166 }
167 const length = bytes[offset + 4]! | (bytes[offset + 5]! << 8);
168 const type = bytes[offset + 6]!;
169 const end = offset + LOG_HEADER_SIZE + length;
170 const fitsBlock = LOG_HEADER_SIZE + length <= blockRemaining;
171 if (fitsBlock && end > bytes.length) break;
172 const typeAndPayload = bytes.subarray(offset + 6, end);
173 if (
174 type === 0 ||
175 !fitsBlock ||
176 crc32c(typeAndPayload) !== unmaskCrc(readUint32(bytes, offset))
177 ) {
178 offset += blockRemaining;
179 pending = null;
180 continue;
181 }
182 const payload = typeAndPayload.subarray(1);
183 offset = end;
184 if (type === 1) {
185 records.push(payload);
186 pending = null;
187 } else if (type === 2) {
188 pending = [payload];
189 } else if (type === 3 && pending) {
190 pending.push(payload);
191 } else if (type === 4 && pending) {
192 pending.push(payload);
193 const total = pending.reduce((sum, part) => sum + part.length, 0);
194 const record = new Uint8Array(total);
195 let position = 0;
196 for (const part of pending) {
197 record.set(part, position);
198 position += part.length;
199 }
200 records.push(record);
201 pending = null;
202 } else {
203 pending = null;
204 }
205 }
206 return records;
207}
208
209interface LiveFiles {
210 readonly tables: ReadonlyArray<number>;
211 readonly logNumber: number;
212 readonly previousLogNumber: number;
213}
214
215/** Replays MANIFEST version edits to find the live table files and log. */
216function readManifest(bytes: Uint8Array): LiveFiles {
217 const tables = new Set<number>();
218 let logNumber = 0;
219 let previousLogNumber = 0;
220 // LevelDB refuses to open a MANIFEST missing any of these, and so do we.
221 const required = new Set([2, 3, 4]);
222 for (const record of readLogRecords(bytes)) {
223 const reader = new Reader(record);
224 while (!reader.done) {
225 const tag = reader.varint();
226 required.delete(tag);
227 switch (tag) {
228 case 1: // comparator
229 reader.lengthPrefixed();
230 break;
231 case 2:
232 logNumber = reader.varint();
233 break;
234 case 3: // next file number
235 case 4: // last sequence
236 reader.varint();
237 break;
238 case 5: // compaction pointer
239 reader.varint();
240 reader.lengthPrefixed();
241 break;
242 case 6: // deleted file
243 reader.varint();
244 tables.delete(reader.varint());
245 break;
246 case 7: {
247 reader.varint();
248 tables.add(reader.varint());
249 reader.varint();
250 reader.lengthPrefixed();
251 reader.lengthPrefixed();
252 break;
253 }
254 case 9:
255 previousLogNumber = reader.varint();
256 break;
257 default:
258 fail(`unknown manifest tag ${tag}`);
259 }
260 }
261 }
262 if (required.size > 0) fail("incomplete manifest");
263 return { tables: [...tables], logNumber, previousLogNumber };
264}
265
266interface VersionedEntry {
267 readonly sequence: number;
268 /** `null` marks a deletion. */
269 readonly value: Uint8Array | null;
270}
271
272type EntryMap = Map<string, { readonly key: Uint8Array; readonly entry: VersionedEntry }>;
273
274const keyId = decodeLatin1;
275
276function record(entries: EntryMap, key: Uint8Array, entry: VersionedEntry) {
277 const id = keyId(key);
278 const existing = entries.get(id);
279 if (!existing || existing.entry.sequence < entry.sequence) entries.set(id, { key, entry });
280}
281
282/** Applies one write-ahead log record (a WriteBatch). */
283function readWriteBatch(entries: EntryMap, batch: Uint8Array) {
284 if (batch.length < 12) fail("write batch too short");
285 // The sequence is a little-endian uint64; real values stay far below 2^53.
286 const sequence = readUint32(batch, 0) + readUint32(batch, 4) * 0x100000000;
287 const count = readUint32(batch, 8);
288 const reader = new Reader(batch, 12);
289 for (let index = 0; index < count; index++) {
290 const type = reader.byte();
291 const key = reader.lengthPrefixed();
292 if (type === 1) {
293 record(entries, key, { sequence: sequence + index, value: reader.lengthPrefixed() });
294 } else if (type === 0) {
295 record(entries, key, { sequence: sequence + index, value: null });
296 } else {
297 fail(`unknown write batch record type ${type}`);
298 }
299 }
300}
301
302function readBlock(table: Uint8Array, reader: Reader): Uint8Array {
303 const offset = reader.varint();
304 const size = reader.varint();
305 if (offset + size + BLOCK_TRAILER_SIZE > table.length) fail("block handle out of range");
306 const contents = table.subarray(offset, offset + size);
307 const compression = table[offset + size]!;
308 const checksum = unmaskCrc(readUint32(table, offset + size + 1));
309 if (crc32c(table.subarray(offset, offset + size + 1)) !== checksum) fail("block checksum");
310 if (compression === 0) return contents;
311 if (compression === 1) return decompressSnappy(contents);
312 return fail(`unsupported block compression ${compression}`);
313}
314
315function* blockEntries(block: Uint8Array): Generator<[Uint8Array, Uint8Array]> {
316 if (block.length < 4) fail("block too short");
317 const restartCount = readUint32(block, block.length - 4);
318 const limit = block.length - 4 - restartCount * 4;
319 if (limit < 0) fail("restart array out of range");
320 const reader = new Reader(block.subarray(0, limit));
321 let key = new Uint8Array(0);
322 while (!reader.done) {
323 const shared = reader.varint();
324 const unshared = reader.varint();
325 const valueLength = reader.varint();
326 if (shared > key.length) fail("block key prefix out of range");
327 const nextKey = new Uint8Array(shared + unshared);
328 nextKey.set(key.subarray(0, shared));
329 nextKey.set(reader.take(unshared), shared);
330 key = nextKey;
331 yield [key, reader.take(valueLength)];
332 }
333}
334
335/** Applies every entry in one sorted table (`.ldb`/`.sst`) file. */
336function readTable(entries: EntryMap, table: Uint8Array) {
337 if (table.length < TABLE_FOOTER_SIZE) fail("table too short");
338 const footer = new Reader(table, table.length - TABLE_FOOTER_SIZE);
339 footer.varint();
340 footer.varint();
341 const index = readBlock(table, footer);
342 for (const [, handle] of blockEntries(index)) {
343 const block = readBlock(table, new Reader(handle));
344 for (const [internalKey, value] of blockEntries(block)) {
345 if (internalKey.length < 8) fail("internal key too short");
346 const trailerOffset = internalKey.length - 8;
347 const trailer =
348 readUint32(internalKey, trailerOffset) +
349 readUint32(internalKey, trailerOffset + 4) * 0x100000000;
350 const type = trailer % 256;
351 const sequence = Math.floor(trailer / 256);
352 const key = internalKey.subarray(0, trailerOffset);
353 record(entries, key, { sequence, value: type === 1 ? value : null });
354 }
355 }
356}
357
358/** Chromium prefixes each stored string with 0 (UTF-16LE) or 1 (Latin-1). */
359function decodeChromiumString(bytes: Uint8Array): string | null {
360 if (bytes.length === 0) return null;
361 const body = bytes.subarray(1);
362 if (bytes[0] === 1) return decodeLatin1(body);
363 if (bytes[0] === 0 && body.length % 2 === 0) return textDecoderUtf16.decode(body);
364 return null;
365}
366
367export class ChromiumLocalStorageReadError extends Schema.TaggedError<ChromiumLocalStorageReadError>()(
368 "ChromiumLocalStorageReadError",
369 {
370 directory: Schema.String,
371 cause: Schema.Defect(),
372 },
373) {
374 override get message() {
375 return `Could not read Chromium Local Storage at ${this.directory}.`;
376 }
377}
378
379/**
380 * Returns the localStorage items one origin (for example `t3code://app`) had
381 * in the profile, or an empty map when the profile never stored any.
382 */
383export const readChromiumLocalStorage = Effect.fn("desktop.chromiumLocalStorage.read")(function* (
384 directory: string,
385 origin: string,
386) {
387 const fs = yield* FileSystem.FileSystem;
388 const path = yield* Path.Path;
389 const readFile = (name: string) => fs.readFile(path.join(directory, name));
390 const wrap = Effect.mapError(
391 (cause: PlatformError.PlatformError | LevelDbFormatError) =>
392 new ChromiumLocalStorageReadError({ directory, cause }),
393 );
394 const decode = <A>(run: () => A) =>
395 Effect.try({
396 try: run,
397 catch: (cause) =>
398 isLevelDbFormatError(cause) ? cause : new LevelDbFormatError({ detail: String(cause) }),
399 });
400
401 const current = textDecoderUtf8.decode(yield* readFile("CURRENT").pipe(wrap)).trim();
402 if (!/^MANIFEST-\d+$/.test(current)) {
403 return yield* new ChromiumLocalStorageReadError({
404 directory,
405 cause: new LevelDbFormatError({ detail: "bad CURRENT" }),
406 });
407 }
408 const manifest = yield* readFile(current).pipe(wrap);
409 const live = yield* decode(() => readManifest(manifest)).pipe(wrap);
410 const logs = (yield* fs.readDirectory(directory).pipe(wrap))
411 .map((name) => /^(\d+)\.log$/.exec(name))
412 // The logs LevelDB's own recovery replays.
413 .filter((match) => {
414 const number = Number(match?.[1]);
415 return number >= live.logNumber || number === live.previousLogNumber;
416 })
417 .map((match) => match![0]);
418
419 const entries: EntryMap = new Map();
420 for (const number of live.tables) {
421 const name = String(number).padStart(6, "0");
422 // Newer LevelDB names tables .ldb; older databases may still hold .sst files.
423 const bytes = yield* readFile(`${name}.ldb`).pipe(
424 Effect.catchIf(
425 (error) => error.reason._tag === "NotFound",
426 () => readFile(`${name}.sst`),
427 ),
428 wrap,
429 );
430 yield* decode(() => readTable(entries, bytes)).pipe(wrap);
431 }
432 for (const name of logs) {
433 const bytes = yield* readFile(name).pipe(wrap);
434 yield* decode(() => {
435 for (const batch of readLogRecords(bytes)) readWriteBatch(entries, batch);
436 }).pipe(wrap);
437 }
438
439 // A flush while another process has the database open can move entries
440 // from a log we read into a table our MANIFEST copy predates. Fail so the
441 // caller retries rather than import a snapshot missing those entries.
442 const currentAfter = textDecoderUtf8.decode(yield* readFile("CURRENT").pipe(wrap)).trim();
443 const manifestAfter = yield* fs.stat(path.join(directory, current)).pipe(wrap);
444 if (currentAfter !== current || Number(manifestAfter.size) !== manifest.length) {
445 return yield* new ChromiumLocalStorageReadError({
446 directory,
447 cause: new LevelDbFormatError({ detail: "database changed while reading" }),
448 });
449 }
450
451 const prefix = `_${origin}\u0000`;
452 const items = new Map<string, string>();
453 for (const [id, { key: rawKey, entry }] of entries) {
454 if (!id.startsWith(prefix) || entry.value === null) continue;
455 const key = decodeChromiumString(rawKey.subarray(prefix.length));
456 const value = decodeChromiumString(entry.value);
457 if (key !== null && value !== null) items.set(key, value);
458 }
459 return items;
460});