diff --git a/extensions/collections/package.json b/extensions/collections/package.json index 738566ae18..be7652fbb6 100644 --- a/extensions/collections/package.json +++ b/extensions/collections/package.json @@ -46,7 +46,7 @@ "shortid": "catalog:", "tmp": "catalog:", "ts-v-gen": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "typescript": "catalog:", "vortex-api": "workspace:*" } diff --git a/extensions/extension-dashlet/package.json b/extensions/extension-dashlet/package.json index 0ffa31980e..fcb5cbaf39 100644 --- a/extensions/extension-dashlet/package.json +++ b/extensions/extension-dashlet/package.json @@ -27,7 +27,7 @@ "react-i18next": "catalog:", "redux": "catalog:", "redux-thunk": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "typescript": "catalog:", "vortex-api": "workspace:*" } diff --git a/extensions/gamebryo-savegame-management/package.json b/extensions/gamebryo-savegame-management/package.json index 11acc4b683..7534fef802 100644 --- a/extensions/gamebryo-savegame-management/package.json +++ b/extensions/gamebryo-savegame-management/package.json @@ -33,7 +33,7 @@ "redux": "catalog:", "redux-act": "catalog:", "redux-thunk": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "vortex-api": "workspace:*", "vortex-parse-ini": "catalog:" }, diff --git a/extensions/games/game-stardewvalley/package.json b/extensions/games/game-stardewvalley/package.json index 0fb2dcb6f7..b98811505c 100644 --- a/extensions/games/game-stardewvalley/package.json +++ b/extensions/games/game-stardewvalley/package.json @@ -26,7 +26,7 @@ "redux-act": "catalog:", "relaxed-json": "catalog:", "semver": "catalog:", - "turbowalk": "catalog:" + "turbowalk": "workspace:*" }, "devDependencies": { "@nexusmods/nexus-api": "catalog:", diff --git a/extensions/gamestore-origin/package.json b/extensions/gamestore-origin/package.json index 863403128e..cd54c9b861 100644 --- a/extensions/gamestore-origin/package.json +++ b/extensions/gamestore-origin/package.json @@ -20,7 +20,7 @@ "@types/libxmljs": "catalog:", "bluebird": "catalog:", "copyfiles": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "typescript": "catalog:", "vortex-api": "workspace:*", "winapi-bindings": "catalog:", diff --git a/extensions/gamestore-xbox/package.json b/extensions/gamestore-xbox/package.json index ebb9a96cb1..e139f4ef89 100644 --- a/extensions/gamestore-xbox/package.json +++ b/extensions/gamestore-xbox/package.json @@ -23,7 +23,7 @@ "copyfiles": "catalog:", "redux": "catalog:", "redux-thunk": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "typescript": "catalog:", "vortex-api": "workspace:*", "winapi-bindings": "catalog:", diff --git a/extensions/mod-content/package.json b/extensions/mod-content/package.json index 361344cf23..d0e1355ede 100644 --- a/extensions/mod-content/package.json +++ b/extensions/mod-content/package.json @@ -26,7 +26,7 @@ "react": "catalog:", "react-bootstrap": "catalog:", "redux": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "vortex-api": "workspace:*" } } diff --git a/extensions/mod-dependency-manager/package.json b/extensions/mod-dependency-manager/package.json index c2d1a7a0fc..617d6386da 100644 --- a/extensions/mod-dependency-manager/package.json +++ b/extensions/mod-dependency-manager/package.json @@ -49,7 +49,7 @@ "reselect": "catalog:", "semver": "catalog:", "shortid": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "vortex-api": "workspace:*" } } diff --git a/extensions/mod-report/package.json b/extensions/mod-report/package.json index 6cbdc2732b..5b3040eb1f 100644 --- a/extensions/mod-report/package.json +++ b/extensions/mod-report/package.json @@ -15,7 +15,7 @@ "bs58": "catalog:", "copyfiles": "catalog:", "electron": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "typescript": "catalog:", "vortex-api": "workspace:*" } diff --git a/extensions/new-file-monitor/package.json b/extensions/new-file-monitor/package.json index 9782e2de85..dee4875b93 100644 --- a/extensions/new-file-monitor/package.json +++ b/extensions/new-file-monitor/package.json @@ -17,7 +17,7 @@ "bluebird": "catalog:", "copyfiles": "catalog:", "redux": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "typescript": "catalog:", "vortex-api": "workspace:*" } diff --git a/extensions/script-extender-error-check/package.json b/extensions/script-extender-error-check/package.json index 02490fece8..83b2afa344 100644 --- a/extensions/script-extender-error-check/package.json +++ b/extensions/script-extender-error-check/package.json @@ -23,7 +23,7 @@ "i18next": "catalog:", "react": "catalog:", "react-bootstrap": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "vortex-api": "workspace:*" } } diff --git a/extensions/test-setup/package.json b/extensions/test-setup/package.json index e865323bd1..7316865799 100644 --- a/extensions/test-setup/package.json +++ b/extensions/test-setup/package.json @@ -19,7 +19,7 @@ "copyfiles": "catalog:", "i18next": "catalog:", "react-bootstrap": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "vortex-api": "workspace:*", "winapi-bindings": "catalog:" } diff --git a/package.json b/package.json index 5320859f4f..75ec428af6 100644 --- a/package.json +++ b/package.json @@ -16,18 +16,18 @@ "prepare": "husky", "preinstall": "npx only-allow pnpm", "start": "pnpm -F @vortex/main run start", - "build": "pnpm run typecheck && pnpm --filter \"@vortex/*\" --filter \"@nexusmods/*\" -r run build", + "build": "pnpm run typecheck && pnpm --filter \"@vortex/*\" --filter \"@nexusmods/*\" --filter \"./packages/**\" --filter \"!@vortex/e2e\" --filter \"!vortex-api\" -r run build", "build:all": "pnpm run build && pnpm run build:extensions && pnpm run build:assets", "build:extensions": "pnpm run api && pnpm run typecheck:extensions && pnpm --filter \"./extensions/**\" run build", "build:assets": "npx tsx scripts/download-duckdb-extensions.ts && node ./scripts/dependency-report.mjs && node ./InstallAssets.mjs ./src/main/out && pnpm sass --style compressed --silence-deprecation=import ./src/stylesheets/loadingScreen.scss ./src/main/out/assets/css/loadingScreen.css && pnpm tailwindcss -i ./src/stylesheets/tailwind-v4.css -o ./src/main/out/assets/css/tailwind-v4.css -m", - "dist": "pnpm run typecheck && pnpm --filter \"@vortex/*\" --filter \"@nexusmods/*\" -r run dist", + "dist": "pnpm run typecheck && pnpm --filter \"@vortex/*\" --filter \"@nexusmods/*\" --filter \"./packages/**\" --filter \"!@vortex/e2e\" --filter \"!vortex-api\" -r run dist", "dist:all": "pnpm run dist && pnpm run dist:extensions && pnpm run dist:assets", "dist:extensions": "pnpm run api && pnpm --filter \"./extensions/**\" run dist", "dist:assets": "npx tsx scripts/download-duckdb-extensions.ts && node ./scripts/dependency-report.mjs && node ./InstallAssets.mjs ./src/main/dist && pnpm sass --style compressed --silence-deprecation=import ./src/stylesheets/loadingScreen.scss ./src/main/dist/assets/css/loadingScreen.css && pnpm tailwindcss -i ./src/stylesheets/tailwind-v4.css -o ./src/main/dist/assets/css/tailwind-v4.css -m", "package": "pnpm run dist:all && pnpm -F @vortex/main run package", "package:nosign": "pnpm run dist:all && pnpm -F @vortex/main run package:nosign", "package:local": "pnpm -F @vortex/main run prepare:win && pnpm run package:nosign", - "typecheck": "pnpm -F @vortex/shared run build && pnpm --filter \"@vortex/*\" --filter \"@nexusmods/*\" -r run typecheck", + "typecheck": "pnpm -F @vortex/shared run build && pnpm --filter \"./packages/**\" --filter \"!@vortex/e2e\" --filter \"!vortex-api\" -r run build && pnpm --filter \"@vortex/*\" --filter \"@nexusmods/*\" --filter \"./packages/**\" --filter \"!@vortex/e2e\" --filter \"!vortex-api\" -r run typecheck", "typecheck:extensions": "pnpm --filter \"./extensions/**\" run typecheck", "lint": "pnpm -r run lint", "lint:quiet": "pnpm -r run lint:quiet", diff --git a/packages/turbowalk/package.json b/packages/turbowalk/package.json new file mode 100644 index 0000000000..e208ab0555 --- /dev/null +++ b/packages/turbowalk/package.json @@ -0,0 +1,18 @@ +{ + "name": "turbowalk", + "version": "4.0.0", + "description": "Fast directory walking using koffi FFI on Windows, fs on Linux", + "main": "src/index.ts", + "types": "src/index.ts", + "private": true, + "type": "commonjs", + "scripts": { + "build": "pnpm tsdown" + }, + "devDependencies": { + "bluebird": "catalog:", + "@types/bluebird": "catalog:", + "koffi": "^2.9.0", + "tsdown": "catalog:" + } +} diff --git a/packages/turbowalk/src/index.test.ts b/packages/turbowalk/src/index.test.ts new file mode 100644 index 0000000000..b70a263048 --- /dev/null +++ b/packages/turbowalk/src/index.test.ts @@ -0,0 +1,146 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; + +import { describe, it, expect, beforeAll, afterAll } from "vitest"; + +import turbowalk, { type IEntry } from "./index"; + +let tmpDir: string; + +beforeAll(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "turbowalk-test-")); + + // Create a directory tree: + // root/ + // file1.txt (10 bytes) + // file2.dat (20 bytes) + // .hidden (0 bytes) + // subdir/ + // nested.txt (5 bytes) + // emptydir/ + // .hiddendir/ + // inside.txt (1 byte) + fs.writeFileSync(path.join(tmpDir, "file1.txt"), "0123456789"); + fs.writeFileSync(path.join(tmpDir, "file2.dat"), "01234567890123456789"); + fs.writeFileSync(path.join(tmpDir, ".hidden"), ""); + fs.mkdirSync(path.join(tmpDir, "subdir")); + fs.writeFileSync(path.join(tmpDir, "subdir", "nested.txt"), "hello"); + fs.mkdirSync(path.join(tmpDir, "emptydir")); + fs.mkdirSync(path.join(tmpDir, ".hiddendir")); + fs.writeFileSync(path.join(tmpDir, ".hiddendir", "inside.txt"), "x"); +}); + +afterAll(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +function collect(dir: string, options?: Parameters[2]): Promise { + const all: IEntry[] = []; + return turbowalk( + dir, + (entries) => { + all.push(...entries); + }, + options, + ).then(() => all); +} + +describe("turbowalk", () => { + it("walks all files and directories recursively", async () => { + const entries = await collect(tmpDir, { skipHidden: false }); + const names = entries.map((e) => path.basename(e.filePath)).sort(); + expect(names).toEqual([ + ".hidden", + ".hiddendir", + "emptydir", + "file1.txt", + "file2.dat", + "inside.txt", + "nested.txt", + "subdir", + ]); + }); + + it("skips hidden files and directories by default", async () => { + const entries = await collect(tmpDir); + const names = entries.map((e) => path.basename(e.filePath)).sort(); + // .hidden, .hiddendir, and inside.txt (inside .hiddendir) should be excluded + expect(names).toEqual(["emptydir", "file1.txt", "file2.dat", "nested.txt", "subdir"]); + }); + + it("reports correct isDirectory flag", async () => { + const entries = await collect(tmpDir, { skipHidden: false }); + const dirs = entries + .filter((e) => e.isDirectory) + .map((e) => path.basename(e.filePath)) + .sort(); + const files = entries + .filter((e) => !e.isDirectory) + .map((e) => path.basename(e.filePath)) + .sort(); + expect(dirs).toEqual([".hiddendir", "emptydir", "subdir"]); + expect(files).toEqual([".hidden", "file1.txt", "file2.dat", "inside.txt", "nested.txt"]); + }); + + it("reports correct file sizes", async () => { + const entries = await collect(tmpDir, { skipHidden: false }); + const file1 = entries.find((e) => path.basename(e.filePath) === "file1.txt"); + const file2 = entries.find((e) => path.basename(e.filePath) === "file2.dat"); + expect(file1?.size).toBe(10); + expect(file2?.size).toBe(20); + }); + + it("reports mtime as unix seconds", async () => { + const entries = await collect(tmpDir, { skipHidden: false }); + const file1 = entries.find((e) => path.basename(e.filePath) === "file1.txt"); + expect(file1?.mtime).toBeGreaterThan(0); + // Should be within a few seconds of now + const now = Math.floor(Date.now() / 1000); + expect(file1!.mtime).toBeGreaterThan(now - 60); + expect(file1!.mtime).toBeLessThanOrEqual(now + 1); + }); + + it("respects recurse: false", async () => { + const entries = await collect(tmpDir, { recurse: false, skipHidden: false }); + const names = entries.map((e) => path.basename(e.filePath)).sort(); + // Should only include top-level entries, not nested.txt or inside.txt + expect(names).toEqual([ + ".hidden", + ".hiddendir", + "emptydir", + "file1.txt", + "file2.dat", + "subdir", + ]); + }); + + it("handles non-existent directory gracefully", async () => { + const entries = await collect(path.join(tmpDir, "nonexistent")); + expect(entries).toEqual([]); + }); + + it("calls progress callback with batched entries", async () => { + const calls: number[] = []; + await turbowalk( + tmpDir, + (entries) => { + calls.push(entries.length); + }, + { skipHidden: false }, + ); + // At least one call should have been made + expect(calls.length).toBeGreaterThan(0); + // Total entries across all calls should match + const total = calls.reduce((a, b) => a + b, 0); + expect(total).toBe(8); // 5 files + 3 dirs + }); + + it("returns full absolute paths", async () => { + const entries = await collect(tmpDir, { skipHidden: false }); + for (const entry of entries) { + expect(path.isAbsolute(entry.filePath)).toBe(true); + expect(entry.filePath.startsWith(tmpDir)).toBe(true); + } + }); +}); diff --git a/packages/turbowalk/src/index.ts b/packages/turbowalk/src/index.ts new file mode 100644 index 0000000000..32cede0041 --- /dev/null +++ b/packages/turbowalk/src/index.ts @@ -0,0 +1,172 @@ +import * as fs from "fs"; +import * as path from "path"; + +import Bluebird from "bluebird"; + +export interface IEntry { + filePath: string; + isDirectory: boolean; + isReparsePoint: boolean; + size: number; + mtime: number; + isTerminator?: boolean; + id?: number; + idStr?: string; + linkCount?: number; +} + +export interface IWalkOptions { + terminators?: boolean; + details?: boolean; + threshold?: number; + recurse?: boolean; + skipHidden?: boolean; + skipLinks?: boolean; + skipInaccessible?: boolean; +} + +const DEFAULTS: Required = { + terminators: false, + details: false, + threshold: 1024, + recurse: true, + skipHidden: true, + skipLinks: true, + skipInaccessible: true, +}; + +// --- Windows: koffi FFI using NtQueryDirectoryFile --- + +let walkDirWindows: + | ((dirPath: string, progress: (entries: IEntry[]) => void, opts: Required) => void) + | undefined; + +if (process.platform === "win32") { + try { + // Dynamic require so it doesn't fail on Linux + const mod = require("./walkWindows") as { + walkDirWindows: typeof walkDirWindows; + }; + walkDirWindows = mod.walkDirWindows; + } catch { + // koffi not available — fall through to TS fallback + } +} + +// --- Linux/fallback: async fs.readdir + concurrent lstat --- + +const STAT_CONCURRENCY = 64; + +async function statBatch( + paths: string[], + opts: Required, +): Promise<(fs.Stats | null)[]> { + const results: (fs.Stats | null)[] = new Array(paths.length); + let cursor = 0; + + async function next(): Promise { + while (cursor < paths.length) { + const idx = cursor++; + try { + results[idx] = await fs.promises.lstat(paths[idx]!); + } catch (err: any) { + if ( + opts.skipInaccessible && + (err?.code === "EACCES" || err?.code === "EPERM" || err?.code === "ENOENT") + ) { + results[idx] = null; + } else { + throw err; + } + } + } + } + + const workers = Math.min(STAT_CONCURRENCY, paths.length); + const tasks: Promise[] = []; + for (let i = 0; i < workers; i++) tasks.push(next()); + await Promise.all(tasks); + return results; +} + +async function walkDirFallback( + dirPath: string, + progress: (entries: IEntry[]) => void, + opts: Required, +): Promise { + let names: string[]; + try { + names = await fs.promises.readdir(dirPath); + } catch (err: any) { + if (err?.code === "ENOENT") return; + if (opts.skipInaccessible && (err?.code === "EACCES" || err?.code === "EPERM")) return; + throw err; + } + + if (opts.skipHidden) { + names = names.filter((n) => !n.startsWith(".")); + } + + const fullPaths = names.map((n) => path.join(dirPath, n)); + const statResults = await statBatch(fullPaths, opts); + + const entries: IEntry[] = []; + const subDirs: string[] = []; + + for (let i = 0; i < names.length; i++) { + const stats = statResults[i]!; + if (stats === null) continue; + + const fullPath = fullPaths[i]!; + const isLink = stats.isSymbolicLink(); + const isDir = stats.isDirectory(); + + entries.push({ + filePath: fullPath, + isDirectory: isDir, + isReparsePoint: isLink, + size: stats.size, + mtime: Math.floor(stats.mtimeMs / 1000), + ...(opts.details ? { id: stats.ino, idStr: String(stats.ino), linkCount: stats.nlink } : {}), + }); + + if (isDir && opts.recurse && !(opts.skipLinks && isLink)) { + subDirs.push(fullPath); + } + } + + if (entries.length > 0) { + progress(entries); + } + + for (const sub of subDirs) { + await walkDirFallback(sub, progress, opts); + } +} + +// --- Public API --- + +function turbowalk( + basePath: string, + progress: (entries: IEntry[]) => void, + options?: IWalkOptions, +): Bluebird { + if (basePath === undefined) { + throw new Error("expected at least one parameter"); + } + const opts = { ...DEFAULTS, ...options }; + const normalized = path.normalize(basePath); + + if (walkDirWindows !== undefined) { + try { + walkDirWindows(normalized, progress, opts); + return Bluebird.resolve(); + } catch (err) { + return Bluebird.reject(err); + } + } + + return Bluebird.resolve(walkDirFallback(normalized, progress, opts)); +} + +export default turbowalk; diff --git a/packages/turbowalk/src/walkWindows.ts b/packages/turbowalk/src/walkWindows.ts new file mode 100644 index 0000000000..cffaaa62f4 --- /dev/null +++ b/packages/turbowalk/src/walkWindows.ts @@ -0,0 +1,371 @@ +/** + * Fast directory walking on Windows using koffi FFI. + * + * Calls NtQueryDirectoryFile (the same NT API the old C++ addon used) which + * returns multiple directory entries per syscall in a packed buffer. This + * avoids the per-entry syscall overhead of FindFirstFileW/FindNextFileW. + */ + +import * as path from "path"; + +import type { IEntry, IWalkOptions } from "./index"; + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const koffi = require("koffi"); + +// --- NT constants --- + +const STATUS_SUCCESS = 0; +const FILE_ATTRIBUTE_DIRECTORY = 0x10; +const FILE_ATTRIBUTE_HIDDEN = 0x2; +const FILE_ATTRIBUTE_REPARSE_POINT = 0x400; + +// CreateFileW constants +const GENERIC_READ = 0x80000000; +const FILE_SHARE_READ = 1; +const FILE_SHARE_WRITE = 2; +const FILE_SHARE_DELETE = 4; +const OPEN_EXISTING = 3; +const FILE_FLAG_BACKUP_SEMANTICS = 0x02000000; +const FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000; +const INVALID_HANDLE_VALUE = BigInt(-1); + +// GetLastError codes +const ERROR_FILE_NOT_FOUND = 2; +const ERROR_PATH_NOT_FOUND = 3; +const ERROR_ACCESS_DENIED = 5; +const ERROR_SHARING_VIOLATION = 32; +const ERROR_LOCK_VIOLATION = 33; + +// NtQueryDirectoryFile information class +const FileFullDirectoryInformation = 2; +const FileAllInformation = 18; + +// FILETIME epoch offset: 100ns ticks between 1601-01-01 and 1970-01-01 +const UNIX_EPOCH_TICKS = 0x019db1ded53e8000n; +const TICKS_PER_SECOND = 10000000n; + +// Buffer for NtQueryDirectoryFile — 1KB matches the old C++ addon's buffer size. +// Counterintuitively, smaller buffers are faster because koffi marshals the +// entire buffer across the FFI boundary each call. 1KB minimizes that overhead +// while still fitting multiple entries per call. +// +// A zero-copy approach using VirtualAlloc + koffi.view() with a 64KB buffer +// was benchmarked and beats even the old C++ addon (~367ms vs 383ms on 89k +// entries). However, Electron's V8 sandbox (v8_enable_sandbox, compile-time +// flag since Electron 21) blocks external ArrayBuffers — koffi.view() throws. +// No runtime flag can disable this. If Vortex ever moves off Electron or the +// sandbox is lifted, switch to VirtualAlloc + koffi.view() + 64KB buffer. +const DIR_BUFFER_SIZE = 1024; + +// Max retries for sharing violations when opening directories +const OPEN_RETRIES = 3; +const RETRY_DELAY_MS = 100; + +// --- koffi function bindings --- + +const kernel32 = koffi.load("kernel32.dll"); +const ntdll = koffi.load("ntdll.dll"); + +// IO_STATUS_BLOCK — used by NtQueryDirectoryFile +const IO_STATUS_BLOCK = koffi.struct("IO_STATUS_BLOCK", { + Status: "int32", + Information: "uintptr", +}); + +const CreateFileW = kernel32.func( + "intptr CreateFileW(const uint16 *lpFileName, uint32 dwDesiredAccess, uint32 dwShareMode, void *lpSecurityAttributes, uint32 dwCreationDisposition, uint32 dwFlagsAndAttributes, intptr hTemplateFile)", +); +const CloseHandle = kernel32.func("bool CloseHandle(intptr hObject)"); +const GetLastError = kernel32.func("uint32 GetLastError()"); +const Sleep = kernel32.func("void Sleep(uint32 dwMilliseconds)"); + +const NtQueryDirectoryFile = ntdll.func( + "int32 NtQueryDirectoryFile(intptr FileHandle, intptr Event, void *ApcRoutine, void *ApcContext, _Out_ IO_STATUS_BLOCK *IoStatusBlock, _Out_ uint8 *FileInformation, uint32 Length, int32 FileInformationClass, bool ReturnSingleEntry, void *FileName, bool RestartScan)", +); + +const NtQueryInformationFile = ntdll.func( + "int32 NtQueryInformationFile(intptr FileHandle, _Out_ IO_STATUS_BLOCK *IoStatusBlock, _Out_ uint8 *FileInformation, uint32 Length, int32 FileInformationClass)", +); + +// --- Helpers --- + +/** Convert a JS string to a null-terminated UTF-16LE buffer. */ +function toWide(str: string): Buffer { + const buf = Buffer.alloc((str.length + 1) * 2); + for (let i = 0; i < str.length; i++) { + buf.writeUInt16LE(str.charCodeAt(i), i * 2); + } + return buf; +} + +/** Apply \\?\ prefix for long path support. */ +function ensureLongPathPrefix(dirPath: string): string { + if (dirPath.startsWith("\\\\?\\") || dirPath.startsWith("\\\\")) { + return dirPath; + } + return "\\\\?\\" + dirPath; +} + +/** + * Open a directory handle for reading. + * Retries on sharing violations (matching C++ behavior). + * Returns INVALID_HANDLE_VALUE on failure. + */ +function openDirectory(dirPath: string): bigint { + const suffix = dirPath.endsWith("\\") ? "" : "\\"; + const widePath = toWide(ensureLongPathPrefix(dirPath) + suffix); + + let handle = INVALID_HANDLE_VALUE; + for (let tries = 0; tries < OPEN_RETRIES; tries++) { + handle = CreateFileW( + widePath, + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + null, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + 0n, + ) as bigint; + + if (handle !== INVALID_HANDLE_VALUE) return handle; + + const err = GetLastError() as number; + if (err !== ERROR_SHARING_VIOLATION && err !== ERROR_LOCK_VIOLATION) { + return INVALID_HANDLE_VALUE; + } + if (tries < OPEN_RETRIES - 1) { + Sleep(RETRY_DELAY_MS); + } + } + return handle; +} + +/** Check whether the last CreateFileW error is recoverable (skip the dir) or fatal. */ +function shouldSkipDirectory(skipInaccessible: boolean): boolean { + const err = GetLastError() as number; + if (err === ERROR_FILE_NOT_FOUND || err === ERROR_PATH_NOT_FOUND) return true; + if (skipInaccessible && err === ERROR_ACCESS_DENIED) return true; + return false; +} + +/** + * Get file details (linkCount, id) for a single file. + * Used only when opts.details is true. + */ +function getFileDetails( + filePath: string, + ioStatus: Record, +): { linkCount: number; id: number; idStr: string } | undefined { + const widePath = toWide(ensureLongPathPrefix(filePath)); + const handle = CreateFileW( + widePath, + 0x80, // FILE_READ_ATTRIBUTES + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + null, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, + 0n, + ) as bigint; + + if (handle === INVALID_HANDLE_VALUE) return undefined; + + try { + // FILE_ALL_INFORMATION is large; we need at least up to InternalInformation + // BasicInformation (40) + StandardInformation (24) + InternalInformation (8) = 72 bytes + const infoBuf = Buffer.alloc(128); + const status = NtQueryInformationFile( + handle, + ioStatus, + infoBuf, + 128, + FileAllInformation, + ) as number; + + if (status !== STATUS_SUCCESS) return undefined; + + // StandardInformation starts at offset 40: NumberOfLinks at +16 (offset 56) + const numberOfLinks = infoBuf.readUInt32LE(56); + // InternalInformation starts at offset 64: IndexNumber (INT64) + const indexLow = infoBuf.readUInt32LE(64); + const indexHigh = infoBuf.readUInt32LE(68); + const indexNumber = (BigInt(indexHigh) << 32n) | BigInt(indexLow >>> 0); + + return { + linkCount: numberOfLinks, + id: Number(indexNumber), + idStr: String(indexNumber), + }; + } finally { + CloseHandle(handle); + } +} + +/** + * Parse FILE_FULL_DIR_INFORMATION entries from the buffer returned by + * NtQueryDirectoryFile. Each entry is variable-length: + * + * ULONG NextEntryOffset (0) + * ULONG FileIndex (4) + * INT64 CreationTime (8) + * INT64 LastAccessTime (16) + * INT64 LastWriteTime (24) + * INT64 ChangeTime (32) + * INT64 EndOfFile (40) + * INT64 AllocationSize (48) + * ULONG FileAttributes (56) + * ULONG FileNameLength (60) + * ULONG EaSize (64) + * WCHAR FileName[1] (68) + */ +function parseEntries( + buf: Buffer, + dirPath: string, + opts: Required, + entries: IEntry[], + subDirs: string[], + ioStatus: Record, +): void { + let offset = 0; + + while (true) { + const nextEntryOffset = buf.readUInt32LE(offset); + const fileAttributes = buf.readUInt32LE(offset + 56); + const fileNameLength = buf.readUInt32LE(offset + 60); + + // Read the filename (UTF-16LE) + const nameStart = offset + 68; + const name = buf.toString("utf16le", nameStart, nameStart + fileNameLength); + + // Skip . and .. + if (name !== "." && name !== "..") { + const isHidden = (fileAttributes & FILE_ATTRIBUTE_HIDDEN) !== 0; + + if (!opts.skipHidden || !isHidden) { + const isDir = (fileAttributes & FILE_ATTRIBUTE_DIRECTORY) !== 0; + const isReparsePoint = (fileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) !== 0; + + // EndOfFile (actual file size) + const sizeLow = buf.readUInt32LE(offset + 40); + const sizeHigh = buf.readUInt32LE(offset + 44); + const size = + sizeHigh === 0 ? sizeLow : Number((BigInt(sizeHigh) << 32n) | BigInt(sizeLow >>> 0)); + + // LastWriteTime → Unix seconds + const wtLow = buf.readUInt32LE(offset + 24); + const wtHigh = buf.readUInt32LE(offset + 28); + const ticks = (BigInt(wtHigh) << 32n) | BigInt(wtLow >>> 0); + const mtime = Number((ticks - UNIX_EPOCH_TICKS) / TICKS_PER_SECOND); + + const fullPath = dirPath + "\\" + name; + + const entry: IEntry = { + filePath: fullPath, + isDirectory: isDir, + isReparsePoint, + size, + mtime, + }; + + if (opts.details) { + const details = getFileDetails(fullPath, ioStatus); + if (details !== undefined) { + entry.linkCount = details.linkCount; + entry.id = details.id; + entry.idStr = details.idStr; + } + } + + entries.push(entry); + + if (isDir && opts.recurse && !(opts.skipLinks && isReparsePoint)) { + subDirs.push(fullPath); + } + } + } + + if (nextEntryOffset === 0) break; + offset += nextEntryOffset; + } +} + +// --- Walk implementation --- + +export function walkDirWindows( + dirPath: string, + progress: (entries: IEntry[]) => void, + opts: Required, +): void { + const allEntries: IEntry[] = []; + // Reuse across all directories to avoid per-directory allocation + const ioStatus: Record = {}; + const buf = Buffer.alloc(DIR_BUFFER_SIZE); + + function walkRecursive(dir: string): void { + const handle = openDirectory(dir); + if (handle === INVALID_HANDLE_VALUE) { + if (!shouldSkipDirectory(opts.skipInaccessible)) { + throw new Error(`Failed to open directory: ${dir}`); + } + return; + } + + const entries: IEntry[] = []; + const subDirs: string[] = []; + + try { + while (true) { + const status = NtQueryDirectoryFile( + handle, + 0n, + null, + null, + ioStatus, + buf, + DIR_BUFFER_SIZE, + FileFullDirectoryInformation, + false, + null, + false, + ) as number; + + if (status === STATUS_SUCCESS) { + parseEntries(buf, dir, opts, entries, subDirs, ioStatus); + } else { + break; + } + } + } finally { + CloseHandle(handle); + } + + for (const entry of entries) { + allEntries.push(entry); + } + + if (allEntries.length >= opts.threshold) { + progress(allEntries.splice(0)); + } + + for (const sub of subDirs) { + walkRecursive(sub); + } + + if (opts.terminators) { + allEntries.push({ + filePath: dir, + isDirectory: true, + isReparsePoint: false, + size: 0, + mtime: 0, + isTerminator: true, + }); + } + } + + walkRecursive(dirPath); + + if (allEntries.length > 0) { + progress(allEntries); + } +} diff --git a/packages/turbowalk/tsdown.config.mts b/packages/turbowalk/tsdown.config.mts new file mode 100644 index 0000000000..7dbd126f49 --- /dev/null +++ b/packages/turbowalk/tsdown.config.mts @@ -0,0 +1,9 @@ +import { defineConfig } from "tsdown"; + +export default defineConfig({ + entry: ["./src/index.ts"], + format: ["cjs"], + platform: "node", + dts: true, + deps: { neverBundle: ["koffi"] }, +}); diff --git a/packages/turbowalk/vitest.config.ts b/packages/turbowalk/vitest.config.ts new file mode 100644 index 0000000000..c1433e6ef3 --- /dev/null +++ b/packages/turbowalk/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + environment: "node", + include: ["src/**/*.test.ts"], + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9a0367055e..7b8e9f9020 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -697,9 +697,6 @@ catalogs: tsdown: specifier: 0.21.0-beta.1 version: 0.21.0-beta.1 - turbowalk: - specifier: git+https://github.com/Nexus-Mods/node-turbowalk#3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f - version: 3.1.1 typed-binary: specifier: ^4.3.3 version: 4.3.3 @@ -1067,8 +1064,8 @@ importers: specifier: 'catalog:' version: 1.0.3 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk typescript: specifier: 'catalog:' version: 5.9.3 @@ -1184,8 +1181,8 @@ importers: specifier: 'catalog:' version: 2.4.2(redux@4.2.1) turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk typescript: specifier: 'catalog:' version: 5.9.3 @@ -1732,8 +1729,8 @@ importers: specifier: 'catalog:' version: 2.4.2(redux@4.2.1) turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk vortex-api: specifier: workspace:* version: link:../../packages/vortex-api @@ -2282,8 +2279,8 @@ importers: specifier: 'catalog:' version: 7.7.4 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../../packages/turbowalk devDependencies: '@nexusmods/nexus-api': specifier: 'catalog:' @@ -2490,8 +2487,8 @@ importers: specifier: 'catalog:' version: 2.4.1 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk typescript: specifier: 'catalog:' version: 5.9.3 @@ -2553,8 +2550,8 @@ importers: specifier: 'catalog:' version: 2.4.2(redux@4.2.1) turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk typescript: specifier: 'catalog:' version: 5.9.3 @@ -2875,8 +2872,8 @@ importers: specifier: 'catalog:' version: 4.2.1 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk vortex-api: specifier: workspace:* version: link:../../packages/vortex-api @@ -2995,8 +2992,8 @@ importers: specifier: 'catalog:' version: 2.2.17 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk vortex-api: specifier: workspace:* version: link:../../packages/vortex-api @@ -3052,8 +3049,8 @@ importers: specifier: 'catalog:' version: 42.0.0 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk typescript: specifier: 'catalog:' version: 5.9.3 @@ -3351,8 +3348,8 @@ importers: specifier: 'catalog:' version: 4.2.1 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk typescript: specifier: 'catalog:' version: 5.9.3 @@ -3540,8 +3537,8 @@ importers: specifier: 'catalog:' version: 0.33.1(react-dom@16.14.0)(react@16.14.0) turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk vortex-api: specifier: workspace:* version: link:../../packages/vortex-api @@ -3679,8 +3676,8 @@ importers: specifier: 'catalog:' version: 0.33.1(react-dom@16.14.0)(react@16.14.0) turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk vortex-api: specifier: workspace:* version: link:../../packages/vortex-api @@ -3840,6 +3837,21 @@ importers: packages/exe-version: {} + packages/turbowalk: + devDependencies: + '@types/bluebird': + specifier: 'catalog:' + version: 3.5.20 + bluebird: + specifier: 'catalog:' + version: 3.7.2 + koffi: + specifier: ^2.9.0 + version: 2.16.1 + tsdown: + specifier: 'catalog:' + version: 0.21.0-beta.1(synckit@0.11.12)(typescript@5.9.3) + packages/vortex-api: dependencies: '@types/bluebird': @@ -4252,8 +4264,8 @@ importers: specifier: 'catalog:' version: 6.0.1 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk universal-analytics: specifier: 'catalog:' version: 0.4.23 @@ -4887,8 +4899,8 @@ importers: specifier: 'catalog:' version: 0.1.0 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk universal-analytics: specifier: 'catalog:' version: 0.4.23 @@ -9297,10 +9309,6 @@ packages: resolution: {integrity: sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==} engines: {node: '>=14.14'} - fs-extra@8.1.0: - resolution: {integrity: sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==} - engines: {node: '>=6 <7 || >=8'} - fs-extra@9.1.0: resolution: {integrity: sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==} engines: {node: '>=10'} @@ -10056,9 +10064,6 @@ packages: engines: {node: '>=6'} hasBin: true - jsonfile@4.0.0: - resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==} - jsonfile@6.2.0: resolution: {integrity: sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==} @@ -10095,6 +10100,9 @@ packages: resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} engines: {node: '>=0.10.0'} + koffi@2.16.1: + resolution: {integrity: sha512-0Ie6CfD026dNfWSosDw9dPxPzO9Rlyo0N8m5r05S8YjytIpuilzMFDMY4IDy/8xQsTwpuVinhncD+S8n3bcYZQ==} + layout-base@1.0.2: resolution: {integrity: sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg==} @@ -12236,10 +12244,6 @@ packages: tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} - turbowalk@https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f: - resolution: {tarball: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f} - version: 3.1.1 - tweetnacl@0.14.5: resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==} @@ -12406,10 +12410,6 @@ packages: universal-analytics@0.4.23: resolution: {integrity: sha512-lgMIH7XBI6OgYn1woDEmxhGdj8yDefMKg7GkWdeATAlQZFrMrNyxSkpDzY57iY0/6fdlzTbBV03OawvvzG+q7A==} - universalify@0.1.2: - resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} - engines: {node: '>= 4.0.0'} - universalify@0.2.0: resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==} engines: {node: '>= 4.0.0'} @@ -17919,12 +17919,6 @@ snapshots: jsonfile: 6.2.0 universalify: 2.0.1 - fs-extra@8.1.0: - dependencies: - graceful-fs: 4.2.11 - jsonfile: 4.0.0 - universalify: 0.1.2 - fs-extra@9.1.0: dependencies: at-least-node: 1.0.0 @@ -18702,10 +18696,6 @@ snapshots: json5@2.2.3: {} - jsonfile@4.0.0: - optionalDependencies: - graceful-fs: 4.2.11 - jsonfile@6.2.0: dependencies: universalify: 2.0.1 @@ -18764,6 +18754,8 @@ snapshots: kind-of@6.0.3: {} + koffi@2.16.1: {} + layout-base@1.0.2: {} lazy-val@1.0.5: {} @@ -21215,13 +21207,6 @@ snapshots: dependencies: safe-buffer: 5.2.1 - turbowalk@https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f: - dependencies: - fs-extra: 8.1.0 - winapi-bindings: https://codeload.github.com/Nexus-Mods/node-winapi-bindings/tar.gz/faa92afe3320731e98abc15b3f5f19c60896d7c1 - transitivePeerDependencies: - - supports-color - tweetnacl@0.14.5: {} type-check@0.4.0: @@ -21409,8 +21394,6 @@ snapshots: transitivePeerDependencies: - supports-color - universalify@0.1.2: {} - universalify@0.2.0: optional: true diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 8d69814426..baaedebab1 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -19,6 +19,7 @@ allowBuilds: drivelist: true electron: true esbuild: true + koffi: true font-scanner: true gamebryo-savegame: true leveldown: true @@ -261,7 +262,6 @@ catalog: ts-v-gen: ^1.0.1 tsconfig-paths-webpack-plugin: ^4.2.0 tsdown: 0.21.0-beta.1 - turbowalk: git+https://github.com/Nexus-Mods/node-turbowalk#3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f typed-binary: ^4.3.3 typescript: 5.9.3 typescript-eslint: 8.47.0 diff --git a/src/main/package.json b/src/main/package.json index d8f30f571a..91df34d7b5 100644 --- a/src/main/package.json +++ b/src/main/package.json @@ -125,7 +125,7 @@ "tailwindcss": "catalog:", "tmp": "catalog:", "tough-cookie": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "universal-analytics": "catalog:", "uuid": "catalog:", "vortex-parse-ini": "catalog:", diff --git a/src/renderer/package.json b/src/renderer/package.json index aba8f7a037..1106b79a18 100644 --- a/src/renderer/package.json +++ b/src/renderer/package.json @@ -169,7 +169,7 @@ "string-template": "catalog:", "tailwindcss": "catalog:", "tmp": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "universal-analytics": "catalog:", "uuid": "catalog:", "vortex-parse-ini": "catalog:", diff --git a/src/renderer/src/extensions/move_activator/index.ts b/src/renderer/src/extensions/move_activator/index.ts index 8e712f5c2e..d0630da260 100644 --- a/src/renderer/src/extensions/move_activator/index.ts +++ b/src/renderer/src/extensions/move_activator/index.ts @@ -252,7 +252,9 @@ class DeploymentMethod extends LinkingDeployment { { details: true, }, - ).then(() => PromiseBB.map(links, (entry) => this.restoreLink(entry.filePath))); + ) + .then(() => PromiseBB.map(links, (entry) => this.restoreLink(entry.filePath))) + .then(() => {}); } protected linkFile(linkPath: string, sourcePath: string, dirTags?: boolean): Promise {