From e53b723631a86aed9fee82f2767511611ccbdd40 Mon Sep 17 00:00:00 2001 From: Zoltan Kochan Date: Fri, 13 Nov 2026 22:35:37 +0100 Subject: [PATCH] fix: support pnpr keeping its registries in the database pnpr 0.1.0-alpha.20 reads its registries only from its database and refuses a config file that declares them, drops `auth.users.file`, and no longer lets its resolver reach the public URL's host at another origin. - Store the registries with `pnpr registries import` before each start. - Drop the users file: accounts live in the database, inside storage, so a seeded storage arrives with the account and `mintToken` signs in to it. - Set a fixed `secret`, which pnpr requires to reuse a cache directory. - Run pnpr on 127.0.0.2 so the resolver can reach its un-proxied address while clients and the latency proxies stay on 127.0.0.1, and allow loopback through `routes.allowedPrivateNetworks`. - Pass `listen` and `publicUrl` in the config so the import checks routes against the same addresses the server uses. Co-Authored-By: Claude Opus 5.5 --- benchmarkRegistry.js | 5 +- latencyProxy.js | 11 +-- pnprServer.js | 166 +++++++++++++++++++++++++++++-------------- populatePnprCache.js | 4 +- 4 files changed, 124 insertions(+), 62 deletions(-) diff --git a/benchmarkRegistry.js b/benchmarkRegistry.js index e436622..e401be2 100644 --- a/benchmarkRegistry.js +++ b/benchmarkRegistry.js @@ -9,6 +9,7 @@ import { installPnpr, mintToken, npmRegistryUrl, + PNPR_HOST, pnprVersion, populateCache, reservePort, @@ -97,8 +98,9 @@ export async function startBenchmarkRegistry ({ managersDirs, fixtureNames }) { // A port the system says is free, rather than a fixed one: a fixed port is // still held by the pnpr of an earlier run in the same job, and that older // server answers for it. - const pnprPort = await reservePort() + const pnprPort = await reservePort(PNPR_HOST) const registryLink = await startLatencyProxy({ + upstreamHost: PNPR_HOST, upstreamPort: pnprPort, roundTripMs: ROUND_TRIP_MS, rateLimit: mbpsToBytesPerSec(BANDWIDTH_MBPS), @@ -115,6 +117,7 @@ export async function startBenchmarkRegistry ({ managersDirs, fixtureNames }) { // Only latency is emulated here: the resolve exchange is small, so what it // costs is round trips rather than throughput. const resolverLink = await startLatencyProxy({ + upstreamHost: PNPR_HOST, upstreamPort: pnprPort, roundTripMs: ROUND_TRIP_MS, logPath: path.join(dir, 'resolver-link.log'), diff --git a/latencyProxy.js b/latencyProxy.js index 70a50dc..257eafc 100644 --- a/latencyProxy.js +++ b/latencyProxy.js @@ -143,7 +143,7 @@ function pump (src, dst, profile, link) { return { flushed: () => queue } } -function listen ({ upstreamPort, roundTripMs, rateLimit, slowStart }) { +function listen ({ upstreamHost, upstreamPort, roundTripMs, rateLimit, slowStart }) { // The profile is one-way; a round trip pays it twice. const profile = { oneWayMs: roundTripMs / 2, rateLimit, slowStart } // One pacing point per direction for the whole proxy: the earliest moment @@ -157,7 +157,7 @@ function listen ({ upstreamPort, roundTripMs, rateLimit, slowStart }) { // finish would close its peer's write side before the chunks still waiting on // the link had been handed over. const server = net.createServer({ allowHalfOpen: true }, (client) => { - const upstream = net.connect({ host: '127.0.0.1', port: upstreamPort, allowHalfOpen: true }) + const upstream = net.connect({ host: upstreamHost, port: upstreamPort, allowHalfOpen: true }) upstream.on('error', () => client.destroy()) client.on('error', () => upstream.destroy()) const toUpstream = pump(client, upstream, profile, uplink) @@ -199,13 +199,14 @@ function listen ({ upstreamPort, roundTripMs, rateLimit, slowStart }) { * manager runs — a proxy living here would accept no connection until the very * install it is supposed to be serving had already finished. */ -export async function startLatencyProxy ({ upstreamPort, roundTripMs, rateLimit = null, slowStart = false, logPath }) { +export async function startLatencyProxy ({ upstreamHost = '127.0.0.1', upstreamPort, roundTripMs, rateLimit = null, slowStart = false, logPath }) { // Like pnpr, the proxy writes to a file instead of a pipe: nothing here // drains a pipe while a measured install holds the event loop, so a full // buffer would block the very process the traffic flows through. const logFd = fs.openSync(logPath, 'a') const proc = spawn(process.execPath, [ path.join(DIRNAME, 'latencyProxy.js'), + `--upstream-host=${upstreamHost}`, `--upstream-port=${upstreamPort}`, `--round-trip-ms=${roundTripMs}`, `--rate-limit=${rateLimit ?? 0}`, @@ -268,9 +269,11 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) { process.on('unhandledRejection', (err) => { console.error(`latency proxy: unhandled rejection: ${err?.stack ?? err}`) }) - const arg = (name) => Number(process.argv.find((a) => a.startsWith(`--${name}=`))?.split('=')[1] ?? 0) + const option = (name) => process.argv.find((value) => value.startsWith(`--${name}=`))?.split('=')[1] + const arg = (name) => Number(option(name) ?? 0) const rateLimit = arg('rate-limit') const server = listen({ + upstreamHost: option('upstream-host') ?? '127.0.0.1', upstreamPort: arg('upstream-port'), roundTripMs: arg('round-trip-ms'), rateLimit: rateLimit > 0 ? rateLimit : null, diff --git a/pnprServer.js b/pnprServer.js index 85f4975..d14b7f0 100644 --- a/pnprServer.js +++ b/pnprServer.js @@ -19,11 +19,11 @@ import { applyPnprServerRegistry, createEnv, pnpmWorkspaceYaml, UPDATED_DEPENDEN * health check, and the run carried on against a registry whose tarball URLs * pointed at a proxy that had been shut down with the run that created it. */ -export function reservePort () { +export function reservePort (host = '127.0.0.1') { return new Promise((resolve, reject) => { const probe = net.createServer() probe.on('error', reject) - probe.listen(0, '127.0.0.1', () => { + probe.listen(0, host, () => { const { port } = probe.address() probe.close(() => resolve(port)) }) @@ -72,6 +72,21 @@ export function tarRegistryUrl (base) { return `${base}/npm/~${TAR_REGISTRY}/` } +/** + * The loopback address pnpr listens on: a host of its own, apart from the + * `127.0.0.1` every client and latency proxy uses. + * + * The accelerated scenario has the resolver fetch metadata from the server's + * own un-proxied address (see `applyPnprServerRegistry`), while the server's + * public URL is the latency proxy in front of it. pnpr refuses to let its + * resolver reach the public URL's host at any other origin, and a hostname + * such as `localhost` is no way around that — the resolver looks it up and + * refuses the address. Listening on another host is what keeps the two + * apart. Every `127.x.x.x` address is loopback on Linux, which is where the + * benchmark runs. + */ +export const PNPR_HOST = '127.0.0.2' + export function installPnpr (managersDir) { // Like pnpm 12 and Bun, the binary arrives through an install script. const result = spawn.sync('pnpm', ['add', '@pnpm/pnpr@next', '--allow-build=@pnpm/pnpr'], { @@ -105,64 +120,44 @@ export async function startPnpr ({ managersDir, dir, port, publicUrl, logLevel = fs.mkdirSync(dir, { recursive: true }) const configPath = path.join(dir, 'pnpr.yaml') fs.writeFileSync(configPath, [ + `listen: ${PNPR_HOST}:${port}`, + // pnpr rewrites the tarball URLs it serves to this address. It has to be + // the address clients actually use, or every tarball would be fetched on a + // link the benchmark isn't emulating — and npm refuses outright to fetch a + // tarball from a host other than the registry it was told about. Set here + // rather than on the command line because the registries import below + // checks its routes against it too. + ...(publicUrl ? [`publicUrl: ${publicUrl}`] : []), + // Accounts and the registries live in the database, which pnpr keeps + // inside the storage directory and binds the storage to. A storage + // `populatePnprCache.js` filled therefore arrives with the populate run's + // account and registries: the account is signed in to again (see + // `mintToken`) and the registries are replaced below. `storage: ${path.join(dir, 'storage')}`, + // Upstream content is cached under keys derived from this, and pnpr + // refuses to reuse a cache directory without one. The benchmark hands + // every run the storage `populatePnprCache.js` filled, so it has to be + // the same in all of them. Nothing here is secret: the server only ever + // listens on loopback. + 'secret: pnpm-benchmarks-pnpr-cache-secret', 'auth:', ' users:', - ` file: ${path.join(dir, 'htpasswd')}`, // The benchmark registers its own resolver user on startup. ' maxUsers: 1', - 'registries:', - // The public npm registry. No `packages:` map, so it claims every - // name — the catch-all, and therefore last in the router's sources. - ' npmjs:', - ' type: upstream', - ' url: https://registry.npmjs.org/', - ' public: true', - // Bit publishes to a registry of its own, and `alotta-packages` - // depends on it. Declaring the scopes it serves is all the routing - // that is needed: every manager keeps installing from the one - // registry URL the benchmark hands it, and the names below are the - // only ones fetched from bit. Tarball URLs come back rewritten to - // pnpr's own address, so bit's packages cross the emulated link - // like every other package. - ' bit:', - ' type: upstream', - ` url: ${BIT_REGISTRY}`, - ' public: true', - ' packages:', - ...BIT_SCOPES.map((scope) => ` '${scope}/*':`), - // One URL in front of both: a router resolves a name to the first - // source whose declared packages claim it. `defaultRegistry` has to - // name *this* rather than a concrete registry — a concrete one - // resolves only its own namespace and answers `Unclaimed` for - // everything else, with no fall-through to a sibling. - ' main:', - ' type: router', - ' sources: [bit, npmjs]', - // The same sources again, served as tars rather than `.tgz` files (see - // `tarRegistryUrl`). One storage behind both: a package pulled through - // either is cached once. - ` ${TAR_REGISTRY}:`, - ' type: router', - ' sources: [bit, npmjs]', - ' tarballs: tar', - 'defaultRegistry: main', 'resolver:', ' enabled: true', - // The accelerated scenario asks the server to resolve against this - // address — its own un-proxied tar registry — so the resolver's metadata fetches - // stay on loopback instead of crossing the emulated link and coming back - // in through the front door (see `applyPnprServerRegistry`). The server - // resolves against whatever registry the client sends but only if that - // registry is declared, so the address has to be listed as a route here. 'routes:', - ' public:', - ` - registry: ${tarRegistryUrl(`http://127.0.0.1:${port}`)}`, + // The tar registry the accelerated scenario resolves against is this + // server's own, at its un-proxied address (see `applyPnprServerRegistry`), + // and the resolver refuses loopback addresses that are not listed here. + ' allowedPrivateNetworks: [127.0.0.0/8]', 'log:', ' format: pretty', ` level: ${logLevel}`, '', ].join('\n')) + const url = `http://${PNPR_HOST}:${port}` + importRegistries({ managersDir, dir, configPath, url }) // pnpr's output goes to a file rather than a pipe. Installs are measured with // a synchronous spawn, so nothing in this process drains a pipe while one is @@ -172,12 +167,6 @@ export async function startPnpr ({ managersDir, dir, port, publicUrl, logLevel = const logFd = fs.openSync(logPath, 'a') const proc = spawn('pnpr', [ '-c', configPath, - '--listen', `127.0.0.1:${port}`, - // pnpr rewrites the tarball URLs it serves to this address. It has to be - // the address clients actually use, or every tarball would be fetched on a - // link the benchmark isn't emulating — and npm refuses outright to fetch a - // tarball from a host other than the registry it was told about. - ...(publicUrl ? ['--public-url', publicUrl] : []), // Packuments are fetched from npmjs once and then treated as authoritative, // so no scenario pays for a revalidation another scenario already paid for. '--packument-ttl', '365d', @@ -204,7 +193,6 @@ export async function startPnpr ({ managersDir, dir, port, publicUrl, logLevel = exit = { code, signal } }) - const url = `http://127.0.0.1:${port}` await waitForPnpr(url, proc, readLog) return { url, @@ -230,6 +218,72 @@ export async function startPnpr ({ managersDir, dir, port, publicUrl, logLevel = } } +/** + * pnpr reads its registries only from its database, never from the config + * file, so they are stored there before the server starts. An import + * replaces whatever an earlier run stored. + */ +function importRegistries ({ managersDir, dir, configPath, url }) { + const registriesPath = path.join(dir, 'registries.yaml') + fs.writeFileSync(registriesPath, [ + 'registries:', + // The public npm registry. No `packages:` map, so it claims every + // name — the catch-all, and therefore last in the router's sources. + ' npmjs:', + ' type: upstream', + ' url: https://registry.npmjs.org/', + ' public: true', + // Bit publishes to a registry of its own, and `alotta-packages` + // depends on it. Declaring the scopes it serves is all the routing + // that is needed: every manager keeps installing from the one + // registry URL the benchmark hands it, and the names below are the + // only ones fetched from bit. Tarball URLs come back rewritten to + // pnpr's own address, so bit's packages cross the emulated link + // like every other package. + ' bit:', + ' type: upstream', + ` url: ${BIT_REGISTRY}`, + ' public: true', + ' packages:', + ...BIT_SCOPES.map((scope) => ` '${scope}/*':`), + // One URL in front of both: a router resolves a name to the first + // source whose declared packages claim it. `defaultRegistry` has to + // name *this* rather than a concrete registry — a concrete one + // resolves only its own namespace and answers `Unclaimed` for + // everything else, with no fall-through to a sibling. + ' main:', + ' type: router', + ' sources: [bit, npmjs]', + // The same sources again, served as tars rather than `.tgz` files (see + // `tarRegistryUrl`). One storage behind both: a package pulled through + // either is cached once. + ` ${TAR_REGISTRY}:`, + ' type: router', + ' sources: [bit, npmjs]', + ' tarballs: tar', + 'defaultRegistry: main', + // The accelerated scenario asks the server to resolve against this + // address — its own un-proxied tar registry — so the resolver's metadata fetches + // stay on loopback instead of crossing the emulated link and coming back + // in through the front door (see `applyPnprServerRegistry`). The server + // resolves against whatever registry the client sends but only if that + // registry is declared, so the address has to be listed as a route here. + 'routes:', + ' public:', + ` - registry: ${tarRegistryUrl(url)}`, + '', + ].join('\n')) + const result = spawn.sync('pnpr', ['-c', configPath, 'registries', 'import', registriesPath], { + cwd: managersDir, + env: createEnv(managersDir), + encoding: 'utf8', + }) + if (result.error) throw result.error + if (result.status !== 0) { + throw new Error(`Importing the pnpr registries failed with status code ${result.status}. ${result.stderr}${result.stdout}`) + } +} + async function waitForPnpr (url, proc, getStderr) { const deadline = Date.now() + 30_000 while (Date.now() < deadline) { @@ -267,7 +321,9 @@ async function waitForPnpr (url, proc, getStderr) { /** * pnpr serves packages anonymously but requires authentication to resolve, so - * the accelerated scenario needs a token. + * the accelerated scenario needs a token. On a storage that already holds the + * account — one `populatePnprCache.js` filled — the same request signs in + * instead of registering. */ export async function mintToken (url) { const res = await fetch(`${url}/-/user/org.couchdb.user:${USERNAME}`, { diff --git a/populatePnprCache.js b/populatePnprCache.js index b3bc1d0..c1056d6 100644 --- a/populatePnprCache.js +++ b/populatePnprCache.js @@ -7,7 +7,7 @@ import tempy from 'tempy' import { cmdsMap } from './commandsMap.js' import { fixtureNames } from './fixtures.js' -import { installPnpr, npmRegistryUrl, populateCache, reservePort, startPnpr, tarRegistryUrl } from './pnprServer.js' +import { installPnpr, npmRegistryUrl, PNPR_HOST, populateCache, reservePort, startPnpr, tarRegistryUrl } from './pnprServer.js' import { bootstrapInstaller, provisionPnpm12 } from './setupPackageManagers.js' const DIRNAME = path.dirname(fileURLToPath(import.meta.url)) @@ -52,7 +52,7 @@ async function run () { installPnpr(managersDirs.pnpr) const dir = path.join(managersDirs.pnpr, 'server') - const port = await reservePort() + const port = await reservePort(PNPR_HOST) const server = await startPnpr({ managersDir: managersDirs.pnpr, dir, port }) try { for (const fixtureName of fixtureNames) {