fix(installer): download verified archives with the identity encoding

.github/workflows/ci.yml

@@ -147,7 +147,7 @@ jobs:
         run: node --experimental-strip-types --test --test-reporter=tap tests/install-wizard.test.ts tests/installer-server.test.ts tests/verify-package-files.test.ts tests/installer-probes.test.ts tests/installer-runner.test.ts tests/installer-preflight.test.ts tests/installer-posix-bootstrap.test.ts tests/installer-windows-bootstrap.test.ts
 
       # The native tests skip off Windows; here every one must run and pass: exactly
-      # the 35 native tests (27 literal tests plus 8 process primitive modes) that
+      # the 36 native tests (28 literal tests plus 8 process primitive modes) that
       # tests/installer-windows-bootstrap.test.ts defines, with 0 skips.
       - name: Require native Windows installer tests
         if: runner.os == 'Windows'
@@ -160,4 +160,4 @@ jobs:
           $text = $output -join "`n"
           $passes = [regex]::Match($text, "(?m)^# pass (\d+)$")
           $skips = [regex]::Match($text, "(?m)^# skipped (\d+)$")
-          if (!$passes.Success -or [int]$passes.Groups[1].Value -ne 35 -or !$skips.Success -or [int]$skips.Groups[1].Value -ne 0) { throw "Native Windows installer tests were skipped or not selected" }
+          if (!$passes.Success -or [int]$passes.Groups[1].Value -ne 36 -or !$skips.Success -or [int]$skips.Groups[1].Value -ne 0) { throw "Native Windows installer tests were skipped or not selected" }

scripts/installer-downloads.mjs

@@ -65,8 +65,16 @@ export function artifactFor(name, platform, arch) {
 }
 
 async function download(descriptor) {
-	const response = await fetch(descriptor.url, { redirect: "error", signal: AbortSignal.timeout(60000) });
-	if (!response.ok || !response.body) throw new Error("Download failed");
+	// identity: fetch would otherwise accept gzip and decode it, so the advertised
+	// length (dl.google.com gzips archives on request) would not match the bytes.
+	const response = await fetch(descriptor.url, { redirect: "error", headers: { "accept-encoding": "identity" },
+		signal: AbortSignal.timeout(60000) });
+	if (!response.ok || !response.body) throw new Error(`Download responded ${response.status}`);
+	const encoding = response.headers.get("content-encoding");
+	if (encoding && encoding.toLowerCase() !== "identity") {
+		await response.body.cancel();
+		throw new Error(`Download was encoded (${encoding}) although identity was requested`);
+	}
 	const advertised = response.headers.get("content-length");
 	if (advertised && (!/^\d+$/.test(advertised) || Number(advertised) > descriptor.maxBytes)) {
 		await response.body.cancel();
@@ -100,7 +108,7 @@ export async function verifiedDownload(name, adapters = {}, platform, arch) {
 	const descriptor = (adapters.artifact ?? artifactFor)(name, platform, arch);
 	let bytes;
 	try { bytes = await (adapters.download ?? download)(descriptor); }
-	catch { throw new Error("Download failed"); }
+	catch (cause) { throw new Error("Download failed", { cause }); }
 	if (!Buffer.isBuffer(bytes) || bytes.length === 0 || bytes.length > descriptor.maxBytes) throw new Error("Download size rejected");
 	if (descriptor.size !== undefined && bytes.length !== descriptor.size) throw new Error("Download size rejected");
 	const [algorithm, encoded] = descriptor.integrity.split("-");

tests/installer-posix-bootstrap.test.ts

@@ -101,6 +101,54 @@ test("download helper fails closed on integrity, empty body and adapter failure"
 		await assert.rejects(verifiedDownload("pnpm", { download }), /Download|integrity/);
 	}
 });
+/** A local server that, like dl.google.com, gzips the body when the client accepts it. */
+async function compressingServer(bytes: Buffer, always = false) {
+	const { createServer } = await import("node:http");
+	const seen: string[] = [];
+	const server = createServer((request, response) => {
+		const accepted = String(request.headers["accept-encoding"] ?? "");
+		seen.push(accepted);
+		if (always || /gzip/.test(accepted)) {
+			const body = gzipSync(bytes);
+			response.writeHead(200, { "content-encoding": "gzip", "content-length": String(body.length) });
+			response.end(body);
+		} else {
+			response.writeHead(200, { "content-length": String(bytes.length) });
+			response.end(bytes);
+		}
+	});
+	await new Promise<void>((done) => server.listen(0, "127.0.0.1", () => done()));
+	const { port } = server.address() as { port: number };
+	return { url: `http://127.0.0.1:${port}/go.zip`, seen, close: () => new Promise((done) => server.close(done)) };
+}
+function localArtifact(url: string, bytes: Buffer) {
+	return () => ({ url, size: bytes.length, maxBytes: bytes.length, integrity: `sha256-${createHash("sha256").update(bytes).digest("hex")}` });
+}
+
+test("the real download asks for the identity encoding, so a server that would gzip the archive sends it whole", async () => {
+	const bytes = Buffer.concat([Buffer.from("PK\u0003\u0004"), Buffer.alloc(4096, 7)]);
+	const server = await compressingServer(bytes);
+	try {
+		const result = await verifiedDownload("go", { artifact: localArtifact(server.url, bytes) });
+		assert.deepEqual(result, bytes);
+		assert.deepEqual(server.seen, ["identity"]);
+	} finally { await server.close(); }
+});
+
+test("an encoded response is refused with its cause instead of being reported as truncated", async () => {
+	const bytes = Buffer.alloc(4096, 9);
+	const server = await compressingServer(bytes, true);
+	try {
+		await assert.rejects(verifiedDownload("go", { artifact: localArtifact(server.url, bytes) }),
+			(error: Error) => error.message === "Download failed" && /encoded/.test(String((error.cause as Error)?.message)));
+	} finally { await server.close(); }
+});
+
+test("a failed download keeps the transport error as its cause", async () => {
+	await assert.rejects(verifiedDownload("pnpm", { download: async () => { throw new Error("socket hang up"); } }),
+		(error: Error) => error.message === "Download failed" && (error.cause as Error)?.message === "socket hang up");
+});
+
 test("download helper validates bytes before returning them", async () => {
 	const bytes = Buffer.from("verified fixture");
 	let observed = "";
@@ -1122,6 +1170,6 @@ test("the CI native Windows gate requires exactly the native tests the Windows b
 	const gate = /\[int\]\$passes\.Groups\[1\]\.Value -ne (\d+) -or/.exec(ci);
 	assert.ok(gate, "the gate compares the passed count exactly");
 	assert.ok(/\[int\]\$skips\.Groups\[1\]\.Value -ne 0/.test(ci), "0 skips stays required");
-	assert.equal(literal + modes, 35);
+	assert.equal(literal + modes, 36);
 	assert.equal(Number(gate[1]), literal + modes);
 });

tests/installer-windows-bootstrap.test.ts

@@ -6,7 +6,7 @@ import { gzipSync } from "node:zlib";
 import { spawn, spawnSync } from "node:child_process";
 import { pathToFileURL } from "node:url";
 import test from "node:test";
-import { artifactFor, compatibleEngine, windowsBootstrapMessage } from "../scripts/installer-downloads.mjs";
+import { acquireGo, artifactFor, compatibleEngine, installedGo, windowsBootstrapMessage } from "../scripts/installer-downloads.mjs";
 import { createProbes, hostAdapters } from "../scripts/installer-probes.mjs";
 import { planPreflight } from "../scripts/installer-preflight.mjs";
 import { lookPath, upgradeInvocation, windowsInvocation } from "../scripts/installer-runner.mjs";
@@ -2122,6 +2122,21 @@ test("native Windows: the main channel extracts its source with System32's tar.e
 // Run 38063142924: from a 179-character package root (a pnpm 11 store path) Go's asm.exe failed
 // with "The directory name is invalid." while building inside the package. The real source
 // build (network: proxy.golang.org and sum.golang.org) from a root deeper than 200 characters.
+test("native Windows: the pinned Go downloads from go.dev, verifies and runs", { skip: nativeUnavailable }, async () => {
+	// The real transport: dl.google.com gzips archives unless identity is requested.
+	const root = mkdtempSync(join(tmpdir(), "gentle pinned go "));
+	try {
+		const result = await acquireGo({ root, platform: "win32", arch: process.arch });
+		assert.equal(result.acquired, true);
+		assert.equal(installedGo(root, "win32", process.arch), result.goPath);
+		const version = spawnSync(result.goPath, ["version"], { encoding: "utf8", env: { ...process.env, GOTOOLCHAIN: "local" } });
+		assert.equal(version.status, 0, version.stderr);
+		assert.match(version.stdout.trim(), new RegExp(`^go version go${result.version} windows/`));
+	} finally {
+		rmSync(root, { recursive: true, force: true });
+	}
+});
+
 test("native Windows: the Gentle AI source build succeeds from a package root deeper than 200 characters", { skip: nativeUnavailable }, async (t) => {
 	const installer = await import("../scripts/gentle-ai-installer.mjs");
 	const where = spawnSync(join(process.env.SystemRoot!, "System32", "where.exe"), ["go.exe"], { encoding: "utf8", windowsHide: true });