authorgravatar for tyler@southboundsoftware.comTyler <tyler@southboundsoftware.com> 2026-08-13 22:07:57-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-08-14 08:37:57+05:30
log7e01d33f0eeb9435299791392d546756cc09c5d3
tree5b1178ed2d857c4b72b4626e37c9063f3c073174
parent9fd788b5a92254a2afa72c2a53513ed2cb730f0d
signature Signed by PGP key B5690EEEBB952194

perf(build): stop unpacking node_modules wholesale from the Windows asar (#5877)

Co-authored-by: tsouth89 <tsouth89@users.noreply.github.com> Co-authored-by: t3-code[bot] <t3-code[bot]@users.noreply.github.com> Co-authored-by: Utkarsh Patil <73941998+UtkarshUsername@users.noreply.github.com> Co-authored-by: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com>

6 files changed, 974 insertions(+), 31 deletions(-)

apps/desktop/src/wsl/DesktopWslEnvironment.ts+19-15
......@@ -229,15 +229,18 @@ const NODE_PTY_PROBE_SCRIPT = (
229229printf 'nodeVersion:%s\\n' "$(node -p 'process.versions.node' 2>/dev/null)"
230230printf 'resolvedPath:%s\\n' "$PATH"
231231cd ${shellQuote(linuxServerDir)} && node <<'NODE' >/dev/null 2>&1
232// The server bundle externalizes its deps to node_modules, and the WSL Node
233// can't read inside app.asar, so confirm those deps are unpacked on the real
234// filesystem before reporting the backend healthy. "effect" is the framework
235// every server module imports; resolving it validates the whole node_modules
236// tree. Exit 3 marks this distinct from a node-pty problem so the caller can
237// report it accurately instead of letting the server crash on
238// ERR_MODULE_NOT_FOUND at launch (which, in wsl-only mode, would just fail to
239// launch with no fallback).
240try { require.resolve("effect"); } catch (_e) { process.exit(3); }
232// The WSL Node can't read inside app.asar, so confirm what the server needs is
233// unpacked on the real filesystem before reporting the backend healthy. Exit 3
234// marks this distinct from a node-pty prebuild problem so the caller can report
235// it accurately instead of letting the server crash on ERR_MODULE_NOT_FOUND at
236// launch (which, in wsl-only mode, would just fail to launch with no fallback).
237//
238// The sentinel must be a package the CLI bundle leaves external. It used to be
239// "effect", back when the bundle externalized its runtime deps and the whole
240// node_modules tree was unpacked. The bundle now inlines its JS dependencies,
241// so "effect" no longer exists on disk and only the native packages do —
242// resolving node-pty is what actually validates the unpacked tree.
243try { require.resolve("node-pty/package.json"); } catch (_e) { process.exit(3); }
241244const fs = require("node:fs");
242245const path = require("node:path");
243246const pkgDir = path.dirname(require.resolve("node-pty/package.json"));
......@@ -462,16 +465,17 @@ const ensureNodePtyImpl = (
462465 } as const;
463466 }
464467
465 // Server dependencies (e.g. "effect") couldn't be resolved on the WSL
466 // filesystem — a packaging regression, since the server bundle needs its
467 // node_modules unpacked from the asar. Fatal so wsl-only mode falls back to
468 // Windows and dual mode surfaces the reason inline, instead of the server
469 // crash-looping on ERR_MODULE_NOT_FOUND once it actually launches.
468 // The packages the server bundle leaves external (node-pty and the other
469 // native addons) couldn't be resolved on the WSL filesystem — a packaging
470 // regression, since those must be unpacked from the asar. Fatal so wsl-only
471 // mode falls back to Windows and dual mode surfaces the reason inline,
472 // instead of the server crash-looping on ERR_MODULE_NOT_FOUND once it
473 // actually launches.
470474 if (probe.exitCode === 3) {
471475 return {
472476 ok: false,
473477 reason:
474 "WSL server dependencies could not be loaded (for example \"effect\"). The server's bundled node_modules is not readable by the WSL distro's Node — this is a packaging problem with this build. Please report it.",
478 'WSL server dependencies could not be loaded (for example "node-pty"). The native packages the server needs are not unpacked where the WSL distro\'s Node can read them — this is a packaging problem with this build. Please report it.',
475479 fatal: true,
476480 } as const;
477481 }
apps/server/vite.config.ts+20-9
......@@ -5,16 +5,20 @@ import baseConfig from "../../vite.config.ts";
55import { loadRepoEnv } from "../../scripts/lib/public-config.ts";
66import packageJson from "./package.json" with { type: "json" };
77
8const bundledPackagePrefixes = [
9 "@pierre/diffs",
10 "@t3tools/",
11 "effect-acp",
12 "effect-codex-app-server",
13];
8// The bundle used to inline only workspace packages, leaving every third-party
9// runtime dep external. External deps must exist on the real filesystem (the WSL
10// backend runs plain `wsl.exe -- node`, which cannot read inside an asar), so the
11// desktop build unpacked `**\/node_modules\/**` wholesale: 13,875 loose files to
12// support 20 native binaries. NSIS install time tracks file count, not bytes.
13//
14// Inverted here — bundle everything except the packages that genuinely cannot be
15// inlined. See scripts/lib/cli-external-packages.ts for what earns an exemption.
16import {
17 isExternalCliDependency,
18 shouldBundleCliDependency,
19} from "../../scripts/lib/cli-external-packages.ts";
1420
15export function shouldBundleCliDependency(id: string): boolean {
16 return bundledPackagePrefixes.some((prefix) => id.startsWith(prefix));
17}
21export { shouldBundleCliDependency };
1822
1923const repoEnv = loadRepoEnv();
2024const cliBuildChannel = packageJson.version.includes("-nightly.") ? "nightly" : "latest";
......@@ -37,7 +41,14 @@ export default mergeConfig(
3741 sourcemap: true,
3842 clean: true,
3943 deps: {
44 // Both halves are required. `alwaysBundle` forces the JS dependencies in
45 // (declared deps are external by default, which is what this change is
46 // undoing). `neverBundle` forces the native packages out: returning
47 // false from `alwaysBundle` only means "no opinion", so a transitive
48 // dependency would still be bundled — which silently inlined
49 // msgpackr-extract and its loader, losing native acceleration.
4050 alwaysBundle: shouldBundleCliDependency,
51 neverBundle: (id: string) => isExternalCliDependency(id),
4152 onlyBundle: false,
4253 },
4354 banner: {
scripts/build-desktop-artifact.test.ts+86
......@@ -1,6 +1,8 @@
11import * as NodeServices from "@effect/platform-node/NodeServices";
22import { assert, it } from "@effect/vitest";
33import * as ConfigProvider from "effect/ConfigProvider";
4import * as FileSystem from "effect/FileSystem";
5import * as Path from "effect/Path";
46import * as Effect from "effect/Effect";
57import * as Layer from "effect/Layer";
68import * as Option from "effect/Option";
......@@ -43,6 +45,8 @@ import {
4345 stageLinuxIconSize,
4446 STAGE_INSTALL_ARGS,
4547 WINDOWS_ASAR_UNPACK,
48 ancestorNodeModulesPaths,
49 copyDirectoryPreservingSymlinks,
4650} from "./build-desktop-artifact.ts";
4751import { BRAND_ASSET_PATHS } from "./lib/brand-assets.ts";
4852import { HostProcessArchitecture, HostProcessPlatform } from "@t3tools/shared/hostProcess";
......@@ -767,3 +771,85 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => {
767771 }),
768772 );
769773});
774
775// The self-containment check runs the packaged tree in a scratch directory. Its
776// own node_modules holds the unpacked externals and must be ignored, but any
777// node_modules *above* it would let Node's parent walk satisfy an import that is
778// missing from the package, so the probe refuses to run in that case.
779it("lists ancestor node_modules, nearest first, excluding the start directory", () => {
780 assert.deepStrictEqual(ancestorNodeModulesPaths("C:\\tmp\\probe\\app", "\\"), [
781 "C:\\tmp\\probe\\node_modules",
782 "C:\\tmp\\node_modules",
783 "C:\\node_modules",
784 ]);
785});
786
787it("includes the filesystem root for posix paths", () => {
788 assert.deepStrictEqual(ancestorNodeModulesPaths("/tmp/probe", "/"), [
789 "/tmp/node_modules",
790 "/node_modules",
791 ]);
792});
793
794// A UNC root must keep its \\server\share prefix. Rebuilding it from segments
795// produced relative paths, which fs.exists resolves against the build cwd, so
796// the guard checked directories that do not exist and silently passed.
797it("keeps the prefix of a UNC path instead of going relative", () => {
798 const paths = ancestorNodeModulesPaths("\\\\server\\share\\tmp\\app", "\\");
799 for (const candidate of paths) {
800 assert.ok(candidate.startsWith("\\\\server\\share"), candidate);
801 }
802 assert.deepStrictEqual(paths[0], "\\\\server\\share\\tmp\\node_modules");
803});
804
805it.effect("rebases packaged links into the isolated tree", () =>
806 Effect.gen(function* () {
807 const fs = yield* FileSystem.FileSystem;
808 const path = yield* Path.Path;
809 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3code-copy-symlinks-" });
810 const source = path.join(root, "source");
811 const destination = path.join(root, "destination");
812 const packageDir = path.join(source, "node_modules/.pnpm/example@1/node_modules/example");
813 const relativePackageLink = path.join(source, "node_modules/example-relative");
814 const absolutePackageLink = path.join(source, "node_modules/example-absolute");
815
816 yield* fs.makeDirectory(packageDir, { recursive: true });
817 yield* fs.writeFileString(path.join(packageDir, "index.js"), "module.exports = true;\n");
818 yield* fs.symlink(
819 path.join(".pnpm", "example@1", "node_modules", "example"),
820 relativePackageLink,
821 );
822 yield* fs.symlink(packageDir, absolutePackageLink);
823
824 yield* copyDirectoryPreservingSymlinks(source, destination);
825
826 const copiedPackage = path.join(
827 destination,
828 "node_modules/.pnpm/example@1/node_modules/example",
829 );
830 const resolvedCopiedPackage = yield* fs.realPath(copiedPackage);
831 assert.equal(
832 yield* fs.readLink(path.join(destination, "node_modules/example-relative")),
833 copiedPackage,
834 );
835 assert.equal(
836 yield* fs.readLink(path.join(destination, "node_modules/example-absolute")),
837 copiedPackage,
838 );
839 assert.equal(
840 yield* fs.realPath(path.join(destination, "node_modules/example-relative")),
841 resolvedCopiedPackage,
842 );
843 assert.equal(
844 yield* fs.realPath(path.join(destination, "node_modules/example-absolute")),
845 resolvedCopiedPackage,
846 );
847 }).pipe(Effect.provide(NodeServices.layer)),
848);
849
850it("ignores trailing separators", () => {
851 assert.deepStrictEqual(
852 ancestorNodeModulesPaths("C:\\tmp\\probe\\app\\", "\\"),
853 ancestorNodeModulesPaths("C:\\tmp\\probe\\app", "\\"),
854 );
855});
scripts/build-desktop-artifact.ts+418-7
......@@ -1,5 +1,7 @@
11#!/usr/bin/env node
2// @effect-diagnostics nodeBuiltinImport:off - Node's typed junction API avoids Windows symlink privileges while keeping the probe isolated.
23
4import * as NodeFSP from "node:fs/promises";
35import * as NodeModule from "node:module";
46
57import { fromYaml } from "@t3tools/shared/schemaYaml";
......@@ -17,17 +19,23 @@ import {
1719 type WebAssetBrand,
1820} from "./lib/brand-assets.ts";
1921import { getDefaultBuildArch } from "./lib/build-target-arch.ts";
22import {
23 CLI_EXTERNAL_PACKAGE_UNPACK_GLOBS,
24 findInlinedExternalPackages,
25} from "./lib/cli-external-packages.ts";
2026import { loadRepoEnv } from "./lib/public-config.ts";
2127import { resolveCatalogDependencies } from "./lib/resolve-catalog.ts";
2228
2329import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
2430import * as NodeServices from "@effect/platform-node/NodeServices";
2531import * as Config from "effect/Config";
32import * as Duration from "effect/Duration";
2633import * as Effect from "effect/Effect";
2734import * as FileSystem from "effect/FileSystem";
2835import * as Layer from "effect/Layer";
2936import * as Logger from "effect/Logger";
3037import * as Option from "effect/Option";
38import type { PlatformError } from "effect/PlatformError";
3139import * as Path from "effect/Path";
3240import * as Schema from "effect/Schema";
3341import * as Stream from "effect/Stream";
......@@ -375,6 +383,52 @@ const desktopBuildInputArtifactNames = {
375383 "bundled-server-client": "bundled server client",
376384} satisfies Record<DesktopBuildInputArtifact, string>;
377385
386/**
387 * Imported by every server module, so it is inlined in any correctly bundled
388 * build. Its absence means the bundle went back to externalizing its
389 * dependencies, which the unpack globs do not cover.
390 */
391const BUNDLE_SELF_CONTAINED_SENTINEL = "effect";
392
393const BUNDLE_SELF_CHECK_TIMEOUT = Duration.seconds(120);
394
395export class ExternalizedBundleError extends Schema.TaggedErrorClass<ExternalizedBundleError>()(
396 "ExternalizedBundleError",
397 { sentinel: Schema.String, inlinedPackageCount: Schema.Number },
398) {
399 override get message(): string {
400 return `The server bundle did not inline "${this.sentinel}" (${this.inlinedPackageCount} packages inlined). The bundle is meant to be self-contained apart from the native externals; if its dependencies are external again they will not be unpacked, and the WSL backend will fail with ERR_MODULE_NOT_FOUND. Check the deps.alwaysBundle wiring in apps/server/vite.config.ts.`;
401 }
402}
403
404export class BundleNotSelfContainedError extends Schema.TaggedErrorClass<BundleNotSelfContainedError>()(
405 "BundleNotSelfContainedError",
406 { exitCode: Schema.Number, output: Schema.String },
407) {
408 override get message(): string {
409 return `The packaged server bundle could not load with only its unpacked dependencies present (exit ${this.exitCode}). Anything it imports that is neither a Node built-in nor an unpacked external is unreachable to the WSL backend, which runs plain node and cannot read app.asar. Output:
410${this.output}`;
411 }
412}
413
414export class InlinedNativePackageError extends Schema.TaggedErrorClass<InlinedNativePackageError>()(
415 "InlinedNativePackageError",
416 { packages: Schema.Array(Schema.String) },
417) {
418 override get message(): string {
419 return `The server bundle inlined packages that load native binaries: ${this.packages.join(", ")}. A node-gyp-build style loader resolves prebuilds relative to its own file, so inlined into a chunk it finds none and the importer quietly falls back to a slower JS path. Add them to CLI_RUNTIME_EXTERNAL_PREFIXES in scripts/lib/cli-external-packages.ts so they stay external and get unpacked.`;
420 }
421}
422
423export class InlinedExternalPackageError extends Schema.TaggedErrorClass<InlinedExternalPackageError>()(
424 "InlinedExternalPackageError",
425 { packages: Schema.Array(Schema.String) },
426) {
427 override get message(): string {
428 return `The server bundle inlined packages that must stay external: ${this.packages.join(", ")}. These are native addons or their loaders; inlined, they resolve prebuilds relative to the bundle and silently lose native acceleration. Check the deps.neverBundle wiring in apps/server/vite.config.ts.`;
429 }
430}
431
378432export class MissingDesktopBuildInputError extends Schema.TaggedErrorClass<MissingDesktopBuildInputError>()(
379433 "MissingDesktopBuildInputError",
380434 {
......@@ -633,13 +687,21 @@ export const DESKTOP_FILE_EXCLUSIONS = [
633687 // are dead weight. The trailing dash keeps the SDK's own JS package.
634688 "!**/node_modules/@anthropic-ai/claude-agent-sdk-*/**/*",
635689] as const;
636// The WSL backend launches the server with plain `wsl.exe -- node`, which
637// cannot read inside an asar archive — and the server bundle externalizes its
638// runtime deps, so the whole node_modules tree must be unpacked, not just the
639// bundle (otherwise ERR_MODULE_NOT_FOUND: "Cannot find package 'effect'").
640// The Windows primary backend reads the same files through the asar redirect,
641// so nothing is duplicated.
642export const WINDOWS_ASAR_UNPACK = ["apps/server/dist/**", "**/node_modules/**"] as const;
690// The WSL backend launches the server with plain `wsl.exe -- node`, which cannot
691// read inside an asar archive, so everything it loads must be on the real
692// filesystem. This used to unpack `**\/node_modules\/**` wholesale, because the
693// server bundle externalized its runtime deps and the Linux Node would fail with
694// ERR_MODULE_NOT_FOUND ("Cannot find package 'effect'") before it even reached
695// node-pty.
696//
697// The CLI bundle now inlines its JS dependencies, so the only things that still
698// have to be loose are the server bundle itself and the packages the bundle
699// leaves external — derived from the same list the bundler uses, so the two
700// cannot drift apart.
701export const WINDOWS_ASAR_UNPACK = [
702 "apps/server/dist/**",
703 ...CLI_EXTERNAL_PACKAGE_UNPACK_GLOBS,
704] as const;
643705export const DESKTOP_EXTRA_RESOURCES = [
644706 {
645707 from: "apps/desktop/prod-resources/resource-monitor",
......@@ -1178,6 +1240,278 @@ const runCommand = Effect.fn("runCommand")(function* (
11781240 }
11791241});
11801242
1243/**
1244 * Every `node_modules` directory that would be visible from `startDir`.
1245 *
1246 * The self-containment check is only meaningful in a directory with none of
1247 * these: Node walks parents when resolving a bare import, so a stray
1248 * node_modules above the probe would satisfy imports that are missing from the
1249 * packaged tree and turn the check into a silent pass.
1250 */
1251function trimTrailingSeparators(value: string): string {
1252 let end = value.length;
1253 while (end > 1 && (value[end - 1] === "/" || value[end - 1] === "\\")) end -= 1;
1254 return value.slice(0, end);
1255}
1256
1257/**
1258 * Length of the `\\server\share` prefix, or 0 when the path is not UNC.
1259 *
1260 * The share is the highest real directory on a UNC path: `\\server` on its own
1261 * is not one, so the ancestor walk must stop there.
1262 */
1263function uncShareRootLength(value: string): number {
1264 const isUnc = value.startsWith("\\\\") || value.startsWith("//");
1265 if (!isUnc) return 0;
1266 const separator = /[\\/]/;
1267 const serverEnd = value.slice(2).search(separator);
1268 if (serverEnd < 0) return value.length;
1269 const shareStart = 2 + serverEnd + 1;
1270 const shareEnd = value.slice(shareStart).search(separator);
1271 return shareEnd < 0 ? value.length : shareStart + shareEnd;
1272}
1273
1274export function ancestorNodeModulesPaths(
1275 startDir: string,
1276 separator: string,
1277): ReadonlyArray<string> {
1278 // Walks with lastIndexOf rather than splitting into segments so UNC roots
1279 // (\\server\share) and drive roots keep their prefix instead of being
1280 // rebuilt into a relative path that silently resolves against the build cwd.
1281 const paths: string[] = [];
1282 let current = trimTrailingSeparators(startDir);
1283 // On a UNC path the share itself is the root: \\server is not a directory, so
1284 // walking past \\server\share would emit paths that cannot exist.
1285 const uncRootLength = uncShareRootLength(current);
1286 for (;;) {
1287 const cut = Math.max(current.lastIndexOf("/"), current.lastIndexOf("\\"));
1288 if (cut < 0 || (uncRootLength > 0 && cut < uncRootLength)) break;
1289 const parent = cut === 0 ? current.slice(0, 1) : current.slice(0, cut);
1290 if (parent === current) break;
1291 paths.push(
1292 parent.endsWith(separator) ? `${parent}node_modules` : `${parent}${separator}node_modules`,
1293 );
1294 if (cut === 0) break;
1295 current = parent;
1296 }
1297 return paths;
1298}
1299
1300const NativeMarkerManifest = Schema.Struct({
1301 dependencies: Schema.optional(Schema.Record(Schema.String, Schema.String)),
1302 optionalDependencies: Schema.optional(Schema.Record(Schema.String, Schema.String)),
1303});
1304const decodeNativeMarkerManifest = Schema.decodeUnknownSync(
1305 Schema.fromJsonString(NativeMarkerManifest),
1306);
1307
1308/** Locate a package inside the pnpm store, which is where the real files live. */
1309const findStorePackageDirectory = Effect.fn("findStorePackageDirectory")(function* (
1310 repoRoot: string,
1311 packageName: string,
1312) {
1313 const fs = yield* FileSystem.FileSystem;
1314 const path = yield* Path.Path;
1315 const storeDir = path.join(repoRoot, "node_modules/.pnpm");
1316 const exists = (candidate: string) =>
1317 fs.exists(candidate).pipe(Effect.orElseSucceed(() => false));
1318 if (!(yield* exists(storeDir))) return null;
1319
1320 const flattened = `${packageName.replace("/", "+")}@`;
1321 const entries = yield* fs
1322 .readDirectory(storeDir)
1323 .pipe(Effect.orElseSucceed(() => [] as string[]));
1324 for (const entry of entries) {
1325 if (!entry.startsWith(flattened)) continue;
1326 const candidate = path.join(storeDir, entry, "node_modules", packageName);
1327 if (yield* exists(candidate)) return candidate;
1328 }
1329 return null;
1330});
1331
1332/** Whether a package builds or ships a native addon it loads at runtime. */
1333const hasNativeLoaderMarkers = Effect.fn("hasNativeLoaderMarkers")(function* (packageDir: string) {
1334 const fs = yield* FileSystem.FileSystem;
1335 const path = yield* Path.Path;
1336 const exists = (candidate: string) =>
1337 fs.exists(candidate).pipe(Effect.orElseSucceed(() => false));
1338
1339 if (yield* exists(path.join(packageDir, "binding.gyp"))) return true;
1340 if (yield* exists(path.join(packageDir, "prebuilds"))) return true;
1341
1342 const manifestPath = path.join(packageDir, "package.json");
1343 if (!(yield* exists(manifestPath))) return false;
1344 const source = yield* fs.readFileString(manifestPath).pipe(Effect.orElseSucceed(() => ""));
1345 if (source === "") return false;
1346 const manifest = yield* Effect.try(() => decodeNativeMarkerManifest(source)).pipe(
1347 Effect.orElseSucceed(() => null),
1348 );
1349 if (manifest === null) return false;
1350 return Object.keys({ ...manifest.dependencies, ...manifest.optionalDependencies }).some(
1351 (dependency) => dependency.startsWith("node-gyp-build"),
1352 );
1353});
1354
1355export const copyDirectoryPreservingSymlinks = Effect.fn("copyDirectoryPreservingSymlinks")(
1356 function* (source: string, destination: string) {
1357 const fs = yield* FileSystem.FileSystem;
1358 const path = yield* Path.Path;
1359
1360 // Effect's Node implementation delegates directory copies to fs.cp, whose
1361 // default rewrites links into absolute source-tree references. Recreate every
1362 // in-tree directory link as a junction rooted in the isolated copy so the
1363 // probe cannot resolve through staging and Windows needs no symlink privilege.
1364 yield* fs.copy(source, destination);
1365
1366 const restoreRelativeSymlinks = (
1367 sourceDirectory: string,
1368 destinationDirectory: string,
1369 ): Effect.Effect<void, PlatformError | BundleNotSelfContainedError> =>
1370 Effect.gen(function* () {
1371 for (const entry of yield* fs.readDirectory(sourceDirectory)) {
1372 const sourceEntry = path.join(sourceDirectory, entry);
1373 const destinationEntry = path.join(destinationDirectory, entry);
1374 const linkTarget = yield* fs.readLink(sourceEntry).pipe(Effect.option);
1375 if (Option.isSome(linkTarget)) {
1376 const absoluteSourceTarget = path.isAbsolute(linkTarget.value)
1377 ? linkTarget.value
1378 : path.resolve(path.dirname(sourceEntry), linkTarget.value);
1379 const sourceRelativeTarget = path.relative(source, absoluteSourceTarget);
1380 if (
1381 sourceRelativeTarget === ".." ||
1382 sourceRelativeTarget.startsWith(`..${path.sep}`) ||
1383 path.isAbsolute(sourceRelativeTarget)
1384 ) {
1385 return yield* new BundleNotSelfContainedError({
1386 exitCode: -1,
1387 output: `Refusing to copy symlink ${sourceEntry}: its target ${absoluteSourceTarget} escapes the packaged tree.`,
1388 });
1389 }
1390 const target = path.join(destination, sourceRelativeTarget);
1391 yield* fs.remove(destinationEntry, { recursive: true, force: true });
1392 yield* Effect.tryPromise({
1393 try: () => NodeFSP.symlink(target, destinationEntry, "junction"),
1394 catch: (cause) =>
1395 new BundleNotSelfContainedError({
1396 exitCode: -1,
1397 output: `Could not isolate ${sourceEntry}: ${String(cause)}`,
1398 }),
1399 });
1400 } else {
1401 const info = yield* fs.stat(sourceEntry);
1402 if (info.type === "Directory") {
1403 yield* restoreRelativeSymlinks(sourceEntry, destinationEntry);
1404 }
1405 }
1406 }
1407 });
1408
1409 yield* restoreRelativeSymlinks(source, destination);
1410 },
1411);
1412
1413const verifyPackagedBundleIsSelfContained = Effect.fn("verifyPackagedBundleIsSelfContained")(
1414 function* (input: { readonly stageDistDir: string; readonly verbose: boolean }) {
1415 const fs = yield* FileSystem.FileSystem;
1416 const path = yield* Path.Path;
1417
1418 // electron-builder names this win-unpacked, win-arm64-unpacked, and so on.
1419 const distEntries = yield* fs
1420 .readDirectory(input.stageDistDir)
1421 .pipe(Effect.orElseSucceed(() => [] as Array<string>));
1422 let unpackedRoot: string | null = null;
1423 for (const entry of distEntries) {
1424 const candidate = path.join(input.stageDistDir, entry, "resources/app.asar.unpacked");
1425 if (yield* fs.exists(candidate).pipe(Effect.orElseSucceed(() => false))) {
1426 unpackedRoot = candidate;
1427 break;
1428 }
1429 }
1430 // Nothing to verify rather than silently passing: a packaging layout change
1431 // should surface here instead of turning the check into a no-op.
1432 if (unpackedRoot === null) {
1433 return yield* new BundleNotSelfContainedError({
1434 exitCode: -1,
1435 output: `No */resources/app.asar.unpacked directory under ${input.stageDistDir}; the bundle self-containment check found nothing to verify.`,
1436 });
1437 }
1438
1439 const probeRoot = yield* fs.makeTempDirectoryScoped({
1440 prefix: "t3code-bundle-selfcheck-",
1441 });
1442 const probeApp = path.join(probeRoot, "app");
1443 yield* copyDirectoryPreservingSymlinks(unpackedRoot, probeApp);
1444
1445 // Guard the guard: if anything above the probe provides a node_modules, a
1446 // missing dependency would resolve there and the check would pass while the
1447 // packaged tree is broken.
1448 for (const candidate of ancestorNodeModulesPaths(probeApp, path.sep)) {
1449 if (yield* fs.exists(candidate).pipe(Effect.orElseSucceed(() => false))) {
1450 return yield* new BundleNotSelfContainedError({
1451 exitCode: -1,
1452 output: `Refusing to report success: ${candidate} is visible from the probe directory, so bare imports could resolve outside the packaged tree. Remove or rename it, or point TMPDIR somewhere without one.`,
1453 });
1454 }
1455 }
1456
1457 const entryPoint = path.join(probeApp, "apps/server/dist/bin.mjs");
1458 if (!(yield* fs.exists(entryPoint).pipe(Effect.orElseSucceed(() => false)))) {
1459 return yield* new BundleNotSelfContainedError({
1460 exitCode: -1,
1461 output: `Expected the server entry at ${entryPoint}.`,
1462 });
1463 }
1464
1465 // --version exercises the eagerly loaded module graph, which is where a
1466 // missing dependency shows up, without starting a server or touching disk
1467 // state. It does not cover lazily imported externals: node-pty is checked
1468 // by the WSL preflight probe at runtime, while ffi-rs, @ff-labs/fff-node
1469 // and the bun adapters are only covered by the unpack globs and the
1470 // inlined-native check below.
1471 yield* runCommand(
1472 ChildProcess.make(
1473 process.execPath,
1474 // --no-global-search-paths because clearing NODE_PATH is not enough:
1475 // CommonJS resolution still falls back to $HOME/.node_modules,
1476 // $HOME/.node_libraries and the install prefix, so a globally installed
1477 // copy of a missing dependency would quietly satisfy this check.
1478 ["--no-global-search-paths", entryPoint, "--version"],
1479 {
1480 cwd: probeApp,
1481 stdout: "pipe",
1482 stderr: "pipe",
1483 // NODE_PATH would let a createRequire call inside the bundle resolve
1484 // a missing external from outside the packaged tree, which is the
1485 // whole thing this is trying to rule out.
1486 env: { ...process.env, NODE_PATH: "" },
1487 },
1488 ),
1489 { label: "bundle self-containment check (node bin.mjs --version)", verbose: input.verbose },
1490 ).pipe(
1491 // Printing a version should be immediate. A regression that blocks (on
1492 // stdin, a port, a lock) would otherwise hang release CI until the job
1493 // times out with nothing useful in the log.
1494 Effect.timeout(BUNDLE_SELF_CHECK_TIMEOUT),
1495 Effect.catchTag("TimeoutError", () =>
1496 Effect.fail(
1497 new BundleNotSelfContainedError({
1498 exitCode: -1,
1499 output: `The packaged bundle did not print its version within ${Duration.toSeconds(BUNDLE_SELF_CHECK_TIMEOUT)}s; it is hanging rather than failing to resolve.`,
1500 }),
1501 ),
1502 ),
1503 Effect.catchTag("BuildCommandFailedError", (error) =>
1504 Effect.fail(
1505 new BundleNotSelfContainedError({
1506 exitCode: error.exitCode,
1507 output: `${error.stderrTail ?? ""}${error.stdoutTail ?? ""}`.trim(),
1508 }),
1509 ),
1510 ),
1511 );
1512 },
1513);
1514
11811515const stageResourceMonitor = Effect.fn("stageResourceMonitor")(function* (input: {
11821516 readonly repoRoot: string;
11831517 readonly stageResourcesDir: string;
......@@ -1817,6 +2151,68 @@ const buildDesktopArtifact = Effect.fn("buildDesktopArtifact")(function* (
18172151 }
18182152 }
18192153
2154 // Assert against the emitted bundle, not the bundler config. `alwaysBundle`
2155 // only forces packages IN, so a transitive dependency of an external package
2156 // is bundled by default however the predicate is written — that silently
2157 // inlined msgpackr-extract and its native loader while every list-based test
2158 // still passed. An inlined native loader resolves its prebuilds relative to
2159 // the bundle and quietly falls back to a slower pure-JS path, so this fails
2160 // the build rather than shipping a silent regression.
2161 {
2162 const chunkNames = (yield* fs.readDirectory(distDirs.serverDist)).filter((entry) =>
2163 entry.endsWith(".mjs"),
2164 );
2165 let totalRegions = 0;
2166 const inlined = new Set<string>();
2167 const inlinedPackages = new Set<string>();
2168 for (const chunkName of chunkNames) {
2169 const source = yield* fs.readFileString(path.join(distDirs.serverDist, chunkName));
2170 const scan = findInlinedExternalPackages(source);
2171 totalRegions += scan.regionCount;
2172 for (const name of scan.inlined) inlined.add(name);
2173 for (const name of scan.inlinedPackages) inlinedPackages.add(name);
2174 }
2175 if (inlined.size > 0) {
2176 return yield* new InlinedExternalPackageError({
2177 packages: [...inlined].sort(),
2178 });
2179 }
2180 // No regions at all means the scan went blind (marker format changed), not
2181 // that the bundle is clean.
2182 if (totalRegions === 0) {
2183 return yield* new InlinedExternalPackageError({
2184 packages: ["<no module regions found; the bundle scan needs updating>"],
2185 });
2186 }
2187 // The check above is one-directional: it only proves nothing external got
2188 // inlined. A regression to externalizing everything would also pass it,
2189 // since source-file regions still exist -- and that is the failure this
2190 // whole change exists to prevent, because those packages are not in the
2191 // unpack globs and the WSL backend would die on ERR_MODULE_NOT_FOUND.
2192 // `effect` is imported by every server module, so it is inlined in any
2193 // correctly bundled build.
2194 // The list-based check above only sees packages someone already thought to
2195 // list. bufferutil and utf-8-validate were inlined for exactly that reason:
2196 // native, but absent from the list, so nothing flagged them. Ask the store
2197 // what each inlined package actually is instead.
2198 const nativeInlined: string[] = [];
2199 for (const name of [...inlinedPackages].sort()) {
2200 const packageDir = yield* findStorePackageDirectory(repoRoot, name);
2201 if (packageDir === null) continue;
2202 if (yield* hasNativeLoaderMarkers(packageDir)) nativeInlined.push(name);
2203 }
2204 if (nativeInlined.length > 0) {
2205 return yield* new InlinedNativePackageError({ packages: nativeInlined });
2206 }
2207
2208 if (!inlinedPackages.has(BUNDLE_SELF_CONTAINED_SENTINEL)) {
2209 return yield* new ExternalizedBundleError({
2210 sentinel: BUNDLE_SELF_CONTAINED_SENTINEL,
2211 inlinedPackageCount: inlinedPackages.size,
2212 });
2213 }
2214 }
2215
18202216 if (!(yield* fs.exists(bundledClientEntry))) {
18212217 return yield* new MissingDesktopBuildInputError({
18222218 artifact: "bundled-server-client",
......@@ -2056,6 +2452,21 @@ const buildDesktopArtifact = Effect.fn("buildDesktopArtifact")(function* (
20562452 });
20572453 }
20582454
2455 // Prove the packaged bundle is self-contained by loading it the way the WSL
2456 // backend does, rather than by reasoning about the emitted source.
2457 //
2458 // Static analysis kept getting this wrong here. Scanning for bare imports
2459 // matched specifiers inside effect's JSDoc examples and inside ajv's runtime
2460 // codegen template, and asserting that one sentinel package was inlined
2461 // missed a build that inlined `effect` while leaving `yaml` external. Node's
2462 // resolver has no such ambiguity: it either finds every import or it does not.
2463 //
2464 // Only Windows unpacks anything; macOS and Linux keep the whole tree inside
2465 // the asar, where this check has nothing to look at.
2466 if (options.platform === "win") {
2467 yield* verifyPackagedBundleIsSelfContained({ stageDistDir, verbose: options.verbose });
2468 }
2469
20592470 const stageEntries = yield* fs.readDirectory(stageDistDir);
20602471 yield* fs.makeDirectory(options.outputDir, { recursive: true });
20612472
scripts/lib/cli-external-packages.test.ts created+275
......@@ -0,0 +1,275 @@
1import * as NodeURL from "node:url";
2
3import * as NodeServices from "@effect/platform-node/NodeServices";
4import { assert, describe, it } from "@effect/vitest";
5import * as Effect from "effect/Effect";
6import * as FileSystem from "effect/FileSystem";
7import * as Path from "effect/Path";
8import * as Schema from "effect/Schema";
9
10import {
11 CLI_EXTERNAL_PACKAGE_PREFIXES,
12 CLI_EXTERNAL_PACKAGE_UNPACK_GLOBS,
13 CLI_RUNTIME_EXTERNAL_PREFIXES,
14 findInlinedExternalPackages,
15 shouldBundleCliDependency,
16} from "./cli-external-packages.ts";
17
18// Only the field this test cares about; decoding ignores everything else.
19// optionalDependencies matter as much as dependencies here: every native family
20// in the list declares its actual platform bindings there (ffi-rs -> @yuuang/*,
21// msgpackr-extract -> @msgpackr-extract/*, fff-node -> @ff-labs/fff-bin-*), so
22// reading only `dependencies` would check nothing for exactly those packages.
23const PackageManifest = Schema.Struct({
24 dependencies: Schema.optional(Schema.Record(Schema.String, Schema.String)),
25 optionalDependencies: Schema.optional(Schema.Record(Schema.String, Schema.String)),
26 peerDependencies: Schema.optional(Schema.Record(Schema.String, Schema.String)),
27});
28type PackageManifest = typeof PackageManifest.Type;
29
30const decodeManifest = Schema.decodeUnknownSync(Schema.fromJsonString(PackageManifest));
31
32describe("shouldBundleCliDependency", () => {
33 it("bundles ordinary runtime dependencies", () => {
34 for (const id of ["effect", "@effect/platform", "hono", "@t3tools/shared/hostProcess"]) {
35 assert.strictEqual(shouldBundleCliDependency(id), true, id);
36 }
37 });
38
39 it("never bundles node: builtins", () => {
40 assert.strictEqual(shouldBundleCliDependency("node:fs"), false);
41 });
42
43 it("leaves native addons and their dlopen wrappers external", () => {
44 for (const id of [
45 "node-pty",
46 "ffi-rs",
47 "@yuuang/ffi-rs-win32-x64-msvc",
48 "@ff-labs/fff-node",
49 "@clerk/electron-passkeys",
50 "msgpackr-extract",
51 "@msgpackr-extract/msgpackr-extract-win32-x64",
52 ]) {
53 assert.strictEqual(shouldBundleCliDependency(id), false, id);
54 }
55 });
56
57 it("leaves bun-only entry points external", () => {
58 assert.strictEqual(shouldBundleCliDependency("@effect/platform-bun"), false);
59 assert.strictEqual(shouldBundleCliDependency("@effect/sql-sqlite-bun"), false);
60 });
61
62 // The real package is `node-gyp-build-optional-packages`, reached by prefix.
63 // Matching it as external while failing to unpack it is invisible on the
64 // Windows primary (which reads app.asar) and breaks only under WSL.
65 it("treats prefix-matched siblings as external", () => {
66 assert.strictEqual(shouldBundleCliDependency("node-gyp-build-optional-packages"), false);
67 });
68});
69
70describe("CLI_EXTERNAL_PACKAGE_UNPACK_GLOBS", () => {
71 it("unpacks every external prefix from both the top level and the pnpm store", () => {
72 for (const prefix of CLI_EXTERNAL_PACKAGE_PREFIXES) {
73 assert.include(CLI_EXTERNAL_PACKAGE_UNPACK_GLOBS, `node_modules/${prefix}*/**/*`, prefix);
74 assert.include(
75 CLI_EXTERNAL_PACKAGE_UNPACK_GLOBS,
76 `node_modules/.pnpm/**/node_modules/${prefix}*/**/*`,
77 prefix,
78 );
79 }
80 });
81
82 // Without the trailing `*` the globs stop covering prefix-matched siblings,
83 // which is exactly how a package ends up external but not unpacked.
84 it("keeps the trailing wildcard that matches prefix siblings", () => {
85 assert.include(CLI_EXTERNAL_PACKAGE_UNPACK_GLOBS, "node_modules/node-gyp-build*/**/*");
86 });
87});
88
89// The failure this guards is invisible on Windows and fatal under WSL.
90//
91// An external package is loaded from the real filesystem, so its own `require`
92// also resolves from the real filesystem. If one of its dependencies was
93// bundled away instead of left external, that dependency exists only inside
94// app.asar — which the Windows primary reads transparently under
95// ELECTRON_RUN_AS_NODE, and plain `node` under WSL cannot.
96//
97// Found the hard way: node-gyp-build-optional-packages requires detect-libc,
98// which was bundled. Windows was fine; WSL got MODULE_NOT_FOUND.
99it.layer(NodeServices.layer)("external package dependency closure", (it) => {
100 // Read manifests off disk from the pnpm store rather than resolving them.
101 // `require("<name>/package.json")` cannot do this job: under pnpm isolation a
102 // transitive package (detect-libc, msgpackr-extract, ffi-rs) is not reachable
103 // by name from this file at all, and an `exports` map can refuse the
104 // `/package.json` subpath outright (@ff-labs/fff-node). Both surface as "not
105 // installed", which would let this test skip everything and pass while
106 // checking nothing. The store is also what asarUnpack globs target, so this
107 // reads the same tree the build packages.
108 const readInstalledPackages = Effect.gen(function* () {
109 const fileSystem = yield* FileSystem.FileSystem;
110 const path = yield* Path.Path;
111 const storeDir = path.resolve(
112 path.dirname(NodeURL.fileURLToPath(import.meta.url)),
113 "../../node_modules/.pnpm",
114 );
115
116 // The store holds regular files too (lock.yaml), so a path built under one
117 // raises ENOTDIR rather than reporting absence. That throws on Linux while
118 // Windows quietly returns false, which is exactly the kind of difference
119 // this test exists to catch, so treat any failure as "not there".
120 const isPresent = (candidate: string) =>
121 fileSystem.exists(candidate).pipe(Effect.orElseSucceed(() => false));
122
123 const installed = new Map<string, PackageManifest>();
124 if (!(yield* isPresent(storeDir))) return installed;
125
126 for (const entry of yield* fileSystem.readDirectory(storeDir)) {
127 const modulesDir = path.join(storeDir, entry, "node_modules");
128 if (!(yield* isPresent(modulesDir))) continue;
129
130 for (const owner of yield* fileSystem.readDirectory(modulesDir)) {
131 const names = owner.startsWith("@")
132 ? (yield* fileSystem.readDirectory(path.join(modulesDir, owner))).map(
133 (scoped) => `${owner}/${scoped}`,
134 )
135 : [owner];
136
137 for (const name of names) {
138 if (installed.has(name)) continue;
139 const manifestPath = path.join(modulesDir, name, "package.json");
140 if (!(yield* isPresent(manifestPath))) continue;
141 installed.set(name, decodeManifest(yield* fileSystem.readFileString(manifestPath)));
142 }
143 }
144 }
145 return installed;
146 }).pipe(Effect.cached, Effect.runSync);
147
148 // Runtime-external only. The build-only entries resolve `bun:*` and are never
149 // loaded by Node, so their closure genuinely does not need to be external.
150 const isRuntimeExternal = (name: string) =>
151 CLI_RUNTIME_EXTERNAL_PREFIXES.some((prefix) => name.startsWith(prefix));
152
153 it.effect("finds the runtime-external packages on disk", () =>
154 Effect.gen(function* () {
155 const installed = yield* readInstalledPackages;
156 const found = [...installed.keys()].filter(isRuntimeExternal);
157
158 // Without this the closure check below can pass vacuously: if nothing is
159 // read, nothing is checked. These are the packages whose closure actually
160 // broke WSL, so require them by name.
161 for (const required of ["node-pty", "node-gyp-build-optional-packages", "detect-libc"]) {
162 assert.ok(
163 found.includes(required),
164 `expected ${required} in the pnpm store; the closure check is only meaningful if it can read these (found ${found.length})`,
165 );
166 }
167 }),
168 );
169
170 it.effect("keeps every runtime dependency of an external package external too", () =>
171 Effect.gen(function* () {
172 const installed = yield* readInstalledPackages;
173 const violations: string[] = [];
174 const seen = new Set<string>();
175 // Seeded from what is actually installed and matches a prefix, so scoped
176 // prefixes like "@yuuang/" and "@ff-labs/" are covered too. Seeding from
177 // the prefix strings themselves would skip every scoped entry, since a
178 // prefix is not a package name.
179 const queue = [...installed.keys()].filter(isRuntimeExternal);
180
181 for (const name of queue) {
182 if (seen.has(name)) continue;
183 seen.add(name);
184
185 const manifest = installed.get(name);
186 if (!manifest) continue;
187
188 const declared = {
189 ...(manifest.dependencies ?? {}),
190 ...(manifest.optionalDependencies ?? {}),
191 ...(manifest.peerDependencies ?? {}),
192 };
193 for (const dependency of Object.keys(declared)) {
194 if (!isRuntimeExternal(dependency)) {
195 violations.push(`${name} -> ${dependency}`);
196 }
197 if (!seen.has(dependency)) queue.push(dependency);
198 }
199 }
200
201 assert.deepStrictEqual(
202 violations,
203 [],
204 `these dependencies of external packages would be bundled away and fail to resolve under WSL: ${violations.join(", ")}`,
205 );
206 }),
207 );
208});
209
210// Configuring the bundler is not the same as checking what it emitted. These
211// exercise the scanner against the marker shape rolldown actually produces.
212describe("findInlinedExternalPackages", () => {
213 const region = (path: string) => `//#region ${path}
214var x = 1;
215//#endregion
216`;
217
218 it("flags an external package that was inlined", () => {
219 const source =
220 region("../../node_modules/.pnpm/detect-libc@2.1.2/node_modules/detect-libc/lib/process.js") +
221 region(
222 "../../node_modules/.pnpm/msgpackr-extract@3.0.4/node_modules/msgpackr-extract/index.js",
223 );
224 const result = findInlinedExternalPackages(source);
225
226 assert.deepStrictEqual(result.inlined, ["detect-libc", "msgpackr-extract"]);
227 assert.strictEqual(result.regionCount, 2);
228 });
229
230 it("flags scoped external packages", () => {
231 const result = findInlinedExternalPackages(
232 region("../../node_modules/@ff-labs/fff-node/dist/src/index.js"),
233 );
234 assert.deepStrictEqual(result.inlined, ["@ff-labs/fff-node"]);
235 });
236
237 it("ignores packages that are meant to be bundled", () => {
238 const source =
239 region("../../node_modules/.pnpm/effect@4.0.0/node_modules/effect/dist/index.js") +
240 region("../../src/server/main.ts");
241 const result = findInlinedExternalPackages(source);
242
243 assert.deepStrictEqual(result.inlined, []);
244 assert.strictEqual(result.regionCount, 2);
245 });
246
247 // regionCount is what separates "clean" from "this scan went blind because the
248 // marker format changed". A caller that ignores it gets a vacuous pass.
249 // The scan has to answer both directions. Checking only that externals are
250 // absent still passes on a bundle that externalized everything, which is the
251 // failure this whole change prevents.
252 it("reports the packages that were inlined, not just the violations", () => {
253 const source =
254 region("../../node_modules/.pnpm/effect@4.0.0/node_modules/effect/dist/index.js") +
255 region("../../node_modules/.pnpm/yaml@2.4.0/node_modules/yaml/dist/index.js") +
256 region("../../src/server/main.ts");
257 const result = findInlinedExternalPackages(source);
258
259 assert.deepStrictEqual(result.inlinedPackages, ["effect", "yaml"]);
260 assert.deepStrictEqual(result.inlined, []);
261 });
262
263 it("does not report the pnpm store directory as a package", () => {
264 const result = findInlinedExternalPackages(
265 region("../../node_modules/.pnpm/effect@4.0.0/node_modules/effect/dist/index.js"),
266 );
267 assert.deepStrictEqual(result.inlinedPackages, ["effect"]);
268 });
269
270 it("reports no regions when the marker format is absent", () => {
271 const result = findInlinedExternalPackages("var x = 1; // node_modules/detect-libc/lib.js");
272 assert.strictEqual(result.regionCount, 0);
273 assert.deepStrictEqual(result.inlined, []);
274 });
275});
scripts/lib/cli-external-packages.ts created+156
......@@ -0,0 +1,156 @@
1/**
2 * The single source of truth for packages the server CLI bundle must NOT inline.
3 *
4 * Two consumers derive from this list, and they must never disagree:
5 *
6 * - apps/server/vite.config.ts decides what stays external to the bundle.
7 * - scripts/build-desktop-artifact.ts decides what gets unpacked out of the asar.
8 *
9 * A package that is external but not unpacked still resolves on the Windows
10 * primary, which runs under ELECTRON_RUN_AS_NODE and reads app.asar
11 * transparently. It fails only under WSL, where the backend is launched as plain
12 * `wsl.exe -- node` and cannot read inside an archive. That asymmetry makes the
13 * drift invisible on the platform you are most likely to test on, which is why
14 * both consumers derive from one list instead of maintaining their own.
15 *
16 * Entries are matched as prefixes (`id.startsWith(prefix)`), so they also cover
17 * a package's platform-specific siblings — `node-gyp-build` covers
18 * `node-gyp-build-optional-packages`, `@yuuang/` covers every `ffi-rs-*` binding.
19 */
20/**
21 * External because Node actually loads them from disk at runtime.
22 *
23 * Native addons (.node), the JS wrappers that dlopen them by real path, and —
24 * critically — the ordinary JS packages those wrappers require. An external
25 * package is loaded from the real filesystem, so its own `require` also
26 * resolves from the real filesystem; a dependency that was bundled away exists
27 * only inside app.asar and is unreachable there. This closure is enforced by a
28 * test, not by inspection.
29 */
30export const CLI_RUNTIME_EXTERNAL_PREFIXES = [
31 "node-pty",
32 "ffi-rs",
33 "@yuuang/",
34 "@ff-labs/",
35 "@clerk/electron-passkeys",
36 "@msgpackr-extract/",
37 "msgpackr-extract",
38 "node-gyp-build",
39 "node-addon-api",
40 // Required by node-gyp-build-optional-packages. Not native, but in the
41 // closure: without it, WSL gets MODULE_NOT_FOUND while Windows is fine.
42 "detect-libc",
43 // ws's optional accelerators. Nothing in this repo declares them, so they are
44 // not in the staged production install and the packaged app does not ship
45 // them either way -- ws wraps the require in try/catch and falls back to its
46 // JS paths. They are listed because they were being inlined from the dev
47 // store: both carry binding.gyp and prebuilds and load through
48 // node-gyp-build, and a native loader inlined into a bundle chunk searches
49 // for prebuilds that cannot be beside it. Listing them keeps that from
50 // becoming real if either is ever declared as a dependency.
51 "bufferutil",
52 "utf-8-validate",
53] as const;
54
55/**
56 * External only so the bundler never has to resolve them.
57 *
58 * These are reached through a runtime-conditional dynamic import that Node
59 * never takes, and they resolve `bun:*` specifiers that do not exist when
60 * bundling for Node. Because Node never loads them, their dependency closure
61 * does not need to be external — only the entry point must stay unbundled.
62 */
63export const CLI_BUILD_ONLY_EXTERNAL_PREFIXES = [
64 "@effect/platform-bun",
65 "@effect/sql-sqlite-bun",
66] as const;
67
68export const CLI_EXTERNAL_PACKAGE_PREFIXES = [
69 ...CLI_RUNTIME_EXTERNAL_PREFIXES,
70 ...CLI_BUILD_ONLY_EXTERNAL_PREFIXES,
71] as const;
72
73/**
74 * True when `id` must stay out of the bundle.
75 *
76 * This has to be wired to the bundler's `neverBundle`, not just to
77 * `alwaysBundle`. `alwaysBundle` only forces packages IN — returning false from
78 * it means "no opinion", and the default then applies: a declared dependency
79 * stays external, but a transitive one gets bundled. That is how
80 * msgpackr-extract, node-gyp-build-optional-packages and detect-libc ended up
81 * inlined while node-pty (a declared dependency) stayed external.
82 */
83export function isExternalCliDependency(id: string): boolean {
84 return CLI_EXTERNAL_PACKAGE_PREFIXES.some((prefix) => id.startsWith(prefix));
85}
86
87/** True when the CLI bundle should inline `id` rather than leave it external. */
88export function shouldBundleCliDependency(id: string): boolean {
89 if (id.startsWith("node:")) return false;
90 return !isExternalCliDependency(id);
91}
92
93/**
94 * asar-unpack globs covering every external package.
95 *
96 * The trailing `*` is what keeps these aligned with the prefix matching above:
97 * without it, `node-gyp-build` would be left external by the bundler and then
98 * not unpacked, because the real package is `node-gyp-build-optional-packages`.
99 *
100 * pnpm stores real files under `.pnpm` and symlinks the top-level names, so both
101 * paths are unpacked for the link target to exist on disk.
102 */
103export const CLI_EXTERNAL_PACKAGE_UNPACK_GLOBS = CLI_EXTERNAL_PACKAGE_PREFIXES.flatMap(
104 (prefix) =>
105 [`node_modules/${prefix}*/**/*`, `node_modules/.pnpm/**/node_modules/${prefix}*/**/*`] as const,
106);
107
108/**
109 * Scan an emitted bundle chunk for runtime-external packages that were inlined.
110 *
111 * Configuring the bundler is not the same as checking what it produced. The
112 * `alwaysBundle` predicate only forces packages IN; returning false from it
113 * means "no opinion", so a transitive dependency still gets bundled by default.
114 * msgpackr-extract, node-gyp-build-optional-packages and detect-libc were
115 * inlined that way while every list-based test passed, which is why this reads
116 * the artifact instead.
117 *
118 * `regionCount` is reported so the caller can tell "nothing was inlined" apart
119 * from "the marker format changed and this scan no longer sees anything".
120 *
121 * `inlinedPackages` is every package seen in a region, which lets the caller
122 * check the opposite direction too. Verifying only that externals are absent
123 * would still pass if the bundler reverted to leaving everything external: the
124 * scan would see source-file regions, report nothing inlined, and the packaged
125 * WSL backend would then fail with ERR_MODULE_NOT_FOUND because those packages
126 * are not in the unpack globs either.
127 */
128export function findInlinedExternalPackages(source: string): {
129 readonly regionCount: number;
130 readonly inlined: ReadonlyArray<string>;
131 readonly inlinedPackages: ReadonlyArray<string>;
132} {
133 // Rolldown marks each inlined module with a `//#region <path>` comment.
134 const regionPattern = /\/\/#region\s+(\S+)/g;
135 const packagePattern = /node_modules\/((?:@[^/\s]+\/)?[^/\s]+)\//g;
136
137 let regionCount = 0;
138 const inlined = new Set<string>();
139 const inlinedPackages = new Set<string>();
140 for (const region of source.matchAll(regionPattern)) {
141 regionCount += 1;
142 const regionPath = region[1] ?? "";
143 for (const candidate of regionPath.matchAll(packagePattern)) {
144 const name = candidate[1];
145 if (name === undefined || name === ".pnpm") continue;
146 inlinedPackages.add(name);
147 if (isExternalCliDependency(name)) inlined.add(name);
148 }
149 }
150
151 return {
152 regionCount,
153 inlined: [...inlined].sort(),
154 inlinedPackages: [...inlinedPackages].sort(),
155 };
156}