From a0b65d3ccd31f13af70617e2d2df115bd1707723 Mon Sep 17 00:00:00 2001 From: Koen Vlaswinkel Date: Tue, 1 Jul 2025 10:50:48 +0200 Subject: [PATCH] Add expansion of short paths using native Windows call --- extensions/ql-vscode/gulpfile.ts/deploy.ts | 5 ++ extensions/ql-vscode/gulpfile.ts/index.ts | 6 +- .../ql-vscode/gulpfile.ts/typescript.ts | 19 +++++- extensions/ql-vscode/package-lock.json | 8 +++ extensions/ql-vscode/package.json | 1 + .../ql-vscode/src/common/short-paths.ts | 65 ++++++++++++++++++- extensions/ql-vscode/src/koffi.d.ts | 4 ++ .../src/variant-analysis/run-remote-query.ts | 42 +++++------- 8 files changed, 116 insertions(+), 34 deletions(-) create mode 100644 extensions/ql-vscode/src/koffi.d.ts diff --git a/extensions/ql-vscode/gulpfile.ts/deploy.ts b/extensions/ql-vscode/gulpfile.ts/deploy.ts index da02b9fd494..7b98600815b 100644 --- a/extensions/ql-vscode/gulpfile.ts/deploy.ts +++ b/extensions/ql-vscode/gulpfile.ts/deploy.ts @@ -4,6 +4,7 @@ import { mkdirs, readdir, unlinkSync, + rename, remove, writeFile, } from "fs-extra"; @@ -45,6 +46,10 @@ async function copyPackage( copyDirectory(resolve(sourcePath, file), resolve(destPath, file)), ), ); + + // The koffi directory needs to be located at the root of the package to ensure + // that the koffi package can find its native modules. + await rename(resolve(destPath, "out", "koffi"), resolve(destPath, "koffi")); } export async function deployPackage(): Promise { diff --git a/extensions/ql-vscode/gulpfile.ts/index.ts b/extensions/ql-vscode/gulpfile.ts/index.ts index d3084561cea..3c20b838c09 100644 --- a/extensions/ql-vscode/gulpfile.ts/index.ts +++ b/extensions/ql-vscode/gulpfile.ts/index.ts @@ -5,7 +5,7 @@ import { checkTypeScript, watchCheckTypeScript, cleanOutput, - copyWasmFiles, + copyModules, } from "./typescript"; import { compileTextMateGrammar } from "./textmate"; import { packageExtension } from "./package"; @@ -21,7 +21,7 @@ export const buildWithoutPackage = series( cleanOutput, parallel( compileEsbuild, - copyWasmFiles, + copyModules, checkTypeScript, compileTextMateGrammar, compileViewEsbuild, @@ -46,7 +46,7 @@ export { watchCheckTypeScript, watchViewEsbuild, compileEsbuild, - copyWasmFiles, + copyModules, checkTypeScript, injectAppInsightsKey, compileViewEsbuild, diff --git a/extensions/ql-vscode/gulpfile.ts/typescript.ts b/extensions/ql-vscode/gulpfile.ts/typescript.ts index abd3591b096..9d49b56821e 100644 --- a/extensions/ql-vscode/gulpfile.ts/typescript.ts +++ b/extensions/ql-vscode/gulpfile.ts/typescript.ts @@ -1,5 +1,5 @@ import { gray, red } from "ansi-colors"; -import { dest, src, watch } from "gulp"; +import { dest, parallel, src, watch } from "gulp"; import esbuild from "gulp-esbuild"; import type { reporter } from "gulp-typescript"; import { createProject } from "gulp-typescript"; @@ -71,7 +71,7 @@ export function watchCheckTypeScript() { watch(["src/**/*.ts", "!src/view/**/*.ts"], checkTypeScript); } -export function copyWasmFiles() { +function copyWasmFiles() { // We need to copy this file for the source-map package to work. Without this fie, the source-map // package is not able to load the WASM file because we are not including the full node_modules // directory. In version 0.7.4, it is not possible to call SourceMapConsumer.initialize in Node environments @@ -83,3 +83,18 @@ export function copyWasmFiles() { encoding: false, }).pipe(dest("out")); } + +function copyNativeAddonFiles() { + // We need to copy these files manually because we only want to include Windows x64 to limit + // the size of the extension. Windows x64 is the most common platform that requires short path + // expansion, so we only include this platform. + // See src/common/short-paths.ts + return pipeline( + src("node_modules/koffi/build/koffi/win32_x64/*.node", { + encoding: false, + }), + dest("out/koffi/win32_x64"), + ); +} + +export const copyModules = parallel(copyWasmFiles, copyNativeAddonFiles); diff --git a/extensions/ql-vscode/package-lock.json b/extensions/ql-vscode/package-lock.json index d513d270c7a..0ceac09dd8b 100644 --- a/extensions/ql-vscode/package-lock.json +++ b/extensions/ql-vscode/package-lock.json @@ -24,6 +24,7 @@ "d3-graphviz": "^5.0.2", "fs-extra": "^11.1.1", "js-yaml": "^4.1.0", + "koffi": "^2.12.0", "msw": "^2.7.4", "nanoid": "^5.0.7", "p-queue": "^8.0.1", @@ -19893,6 +19894,13 @@ "node": ">=6" } }, + "node_modules/koffi": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/koffi/-/koffi-2.12.0.tgz", + "integrity": "sha512-J886y/bvoGG4ZhMVstB2Nh6/q9tzAYn0kaH7Ss8DWavGIxP5jOLzUY9IZzw9pMuXArj0SLSpl0MYsKRURPAv7g==", + "hasInstallScript": true, + "license": "MIT" + }, "node_modules/language-subtag-registry": { "version": "0.3.22", "resolved": "https://registry.npmjs.org/language-subtag-registry/-/language-subtag-registry-0.3.22.tgz", diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 4f4e0079250..e38e8db0c81 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -1995,6 +1995,7 @@ "d3-graphviz": "^5.0.2", "fs-extra": "^11.1.1", "js-yaml": "^4.1.0", + "koffi": "^2.12.0", "msw": "^2.7.4", "nanoid": "^5.0.7", "p-queue": "^8.0.1", diff --git a/extensions/ql-vscode/src/common/short-paths.ts b/extensions/ql-vscode/src/common/short-paths.ts index 838dac031e1..d60d6231458 100644 --- a/extensions/ql-vscode/src/common/short-paths.ts +++ b/extensions/ql-vscode/src/common/short-paths.ts @@ -1,7 +1,9 @@ -import { platform } from "os"; +import { arch, platform } from "os"; import { basename, dirname, join, normalize, resolve } from "path"; import { lstat, readdir } from "fs/promises"; import type { BaseLogger } from "./logging"; +import type { KoffiFunction } from "koffi"; +import { getErrorMessage } from "./helpers-pure"; /** * Expands a path that potentially contains 8.3 short names (e.g. "C:\PROGRA~1" instead of "C:\Program Files"). @@ -32,7 +34,23 @@ export async function expandShortPaths( return absoluteShortPath; } - return await expandShortPathRecursive(absoluteShortPath, logger); + const longPath = await expandShortPathRecursive(absoluteShortPath, logger); + if (longPath.indexOf("~") < 0) { + return longPath; + } + + void logger.log( + "Short path was not resolved to long path, using native method", + ); + + try { + return await expandShortPathNative(absoluteShortPath, logger); + } catch (e: unknown) { + void logger.log( + `Failed to expand short path using native method: ${getErrorMessage(e)}`, + ); + return longPath; + } } /** @@ -115,3 +133,46 @@ async function expandShortPathRecursive( const longBase = await expandShortPathComponent(dir, shortBase, logger); return join(dir, longBase); } + +let GetLongPathNameW: KoffiFunction | undefined; + +async function expandShortPathNative(shortPath: string, logger: BaseLogger) { + if (platform() !== "win32") { + throw new Error("expandShortPathNative is only supported on Windows"); + } + + if (arch() !== "x64") { + throw new Error( + "expandShortPathNative is only supported on x64 architecture", + ); + } + + if (GetLongPathNameW === undefined) { + // We are using koffi/indirect here to avoid including the native addon for all + // platforms in the bundle since this is only used on Windows. Instead, the + // native addon is included in the Gulpfile. + const koffi = await import("koffi/indirect"); + + const lib = koffi.load("kernel32.dll"); + GetLongPathNameW = lib.func("__stdcall", "GetLongPathNameW", "uint32", [ + "str16", + "str16", + "uint32", + ]); + } + + const MAX_PATH = 32767; + const buffer = Buffer.alloc(MAX_PATH * 2, 0); + + const result = GetLongPathNameW(shortPath, buffer, MAX_PATH); + + if (result === 0) { + throw new Error("Failed to get long path name"); + } + + const longPath = buffer.toString("utf16le", 0, (result - 1) * 2); + + void logger.log(`Expanded short path ${shortPath} to ${longPath}`); + + return longPath; +} diff --git a/extensions/ql-vscode/src/koffi.d.ts b/extensions/ql-vscode/src/koffi.d.ts new file mode 100644 index 00000000000..48cabce7265 --- /dev/null +++ b/extensions/ql-vscode/src/koffi.d.ts @@ -0,0 +1,4 @@ +// koffi/indirect is untyped in the upstream package, but it exports the same functions as koffi. +declare module "koffi/indirect" { + export * from "koffi"; +} diff --git a/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts b/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts index c49d9d3f27e..e7e67a0df6c 100644 --- a/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts +++ b/extensions/ql-vscode/src/variant-analysis/run-remote-query.ts @@ -2,9 +2,8 @@ import type { CancellationToken } from "vscode"; import { Uri, window } from "vscode"; import { join, sep, basename, relative } from "path"; import { dump, load } from "js-yaml"; -import { copy, writeFile, readFile, mkdirp } from "fs-extra"; -import type { DirectoryResult } from "tmp-promise"; -import { dir, tmpName } from "tmp-promise"; +import { copy, writeFile, readFile, mkdirp, remove } from "fs-extra"; +import { nanoid } from "nanoid"; import { tmpDir } from "../tmp-dir"; import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders"; import type { Credentials } from "../common/authentication"; @@ -236,39 +235,28 @@ async function copyExistingQueryPack( } interface RemoteQueryTempDir { - remoteQueryDir: DirectoryResult; + remoteQueryDir: string; queryPackDir: string; compiledPackDir: string; bundleFile: string; } async function createRemoteQueriesTempDirectory(): Promise { - const shortRemoteQueryDir = await dir({ - dir: tmpDir.name, - unsafeCleanup: true, - }); // Expand 8.3 filenames here to work around a CLI bug where `codeql pack bundle` produces an empty // archive if the pack path contains any 8.3 components. - const remoteQueryDir = { - ...shortRemoteQueryDir, - path: await expandShortPaths(shortRemoteQueryDir.path, extLogger), - }; - const queryPackDir = join(remoteQueryDir.path, "query-pack"); + const tmpDirPath = await expandShortPaths(tmpDir.name, extLogger); + + const remoteQueryDir = join(tmpDirPath, `remote-query-${nanoid()}`); + await mkdirp(remoteQueryDir); + + const queryPackDir = join(remoteQueryDir, "query-pack"); await mkdirp(queryPackDir); - const compiledPackDir = join(remoteQueryDir.path, "compiled-pack"); - const bundleFile = await expandShortPaths( - await getPackedBundlePath(tmpDir.name), - extLogger, - ); - return { remoteQueryDir, queryPackDir, compiledPackDir, bundleFile }; -} -async function getPackedBundlePath(remoteQueryDir: string): Promise { - return tmpName({ - dir: remoteQueryDir, - postfix: "generated.tgz", - prefix: "qlpack", - }); + const compiledPackDir = join(remoteQueryDir, "compiled-pack"); + + const bundleFile = join(remoteQueryDir, `qlpack-${nanoid()}-generated.tgz`); + + return { remoteQueryDir, queryPackDir, compiledPackDir, bundleFile }; } interface PreparedRemoteQuery { @@ -337,7 +325,7 @@ export async function prepareRemoteQueryRun( token, ); } finally { - await tempDir.remoteQueryDir.cleanup(); + await remove(tempDir.remoteQueryDir); } if (token.isCancellationRequested) {