Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,3 +56,11 @@ That section times commands inside the shell with `$EPOCHREALTIME`, so it needs
```
brew install bash
```

## Lockfile Git merge conflicts

`benchmarkLockfileConflicts.js` measures 3-way `git merge-file` conflicts and semantic drift across 25 concurrent-branch scenarios (`lockfileConflictScenarios.js`) for `pnpm-lock.yaml` (`13.0` and `9.0`), `package-lock.json` (npm v3), `yarn.lock` (Berry and Classic v1), `bun.lock`, and `vlt-lock.json`, and writes the results to [lockfile-conflicts.json](./lockfile-conflicts.json):

```
pnpm run benchmark:conflicts
```
218 changes: 218 additions & 0 deletions benchmarkLockfileConflicts.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,218 @@
'use strict'
import fs from 'node:fs'
import os from 'node:os'
import path from 'node:path'
import { fileURLToPath, pathToFileURL } from 'node:url'

import spawn from 'cross-spawn'

import { lockfileConflictScenarios } from './lockfileConflictScenarios.js'

const DIRNAME = path.dirname(fileURLToPath(import.meta.url))
const OUTPUT_FILE = path.join(DIRNAME, 'lockfile-conflicts.json')

export const CONFLICT_MANAGERS = [
'pnpm_v13',
'pnpm_v9',
'npm',
'yarn',
'yarn_classic',
'bun',
'vlt',
]

function rootManifest (state) {
return {
name: 'root',
version: '1.0.0',
private: true,
...(state.workspaces ? { workspaces: ['packages/*'] } : {}),
...(state.dependencies ? { dependencies: state.dependencies } : {}),
...(state.devDependencies ? { devDependencies: state.devDependencies } : {}),
}
}

export function writeProjectState (dir, state, { clean = true } = {}) {
if (clean) {
fs.rmSync(dir, { recursive: true, force: true })
fs.mkdirSync(dir, { recursive: true })
}
fs.writeFileSync(path.join(dir, 'package.json'), `${JSON.stringify(rootManifest(state), null, 2)}\n`)
if (!state.workspaces) return
for (const [wsName, wsPkg] of Object.entries(state.workspaces)) {
const wsDir = path.join(dir, 'packages', wsName)
fs.mkdirSync(wsDir, { recursive: true })
const manifest = { name: `@ws/${wsName}`, version: '1.0.0', ...wsPkg }
fs.writeFileSync(path.join(wsDir, 'package.json'), `${JSON.stringify(manifest, null, 2)}\n`)
}
}

function writePnpmWorkspaceYaml (dir, state, isV13) {
const lines = state.workspaces ? ['packages:', ' - packages/*'] : []
if (isV13) lines.push('lockfile:', ' format: v13')
if (lines.length > 0) {
fs.writeFileSync(path.join(dir, 'pnpm-workspace.yaml'), `${lines.join('\n')}\n`)
}
}

function runCommand (command, args, opts) {
const result = spawn.sync(command, args, { stdio: 'pipe', ...opts })
if (result.error) throw result.error
if (result.status !== 0) {
const stderr = result.stderr?.toString() ?? ''
throw new Error(`\`${command} ${args.join(' ')}\` exited with ${result.status}: ${stderr}`)
}
}

function invokeManager (manager, dir, binaries) {
if (manager === 'pnpm_v13' || manager === 'pnpm_v9') {
runCommand(binaries.pnpm ?? 'pnpm', ['install', '--lockfile-only', '--ignore-scripts'], { cwd: dir })
return fs.readFileSync(path.join(dir, 'pnpm-lock.yaml'), 'utf8')
}
if (manager === 'npm') {
runCommand(binaries.npm ?? 'npm', ['install', '--package-lock-only', '--ignore-scripts', '--no-audit', '--no-fund'], { cwd: dir })
return fs.readFileSync(path.join(dir, 'package-lock.json'), 'utf8')
}
if (manager === 'bun') {
runCommand(binaries.bun ?? 'bun', ['install', '--lockfile-only', '--ignore-scripts'], { cwd: dir })
return fs.readFileSync(path.join(dir, 'bun.lock'), 'utf8')
}
if (manager === 'vlt') {
const args = ['install', '--lockfile-only', '--registry=https://registry.npmjs.org/', '--registries=npm=https://registry.npmjs.org/']
runCommand(binaries.vlt ?? 'vlt', args, { cwd: dir })
return fs.readFileSync(path.join(dir, 'vlt-lock.json'), 'utf8')
}
if (manager === 'yarn') {
const env = { ...process.env, YARN_ENABLE_IMMUTABLE_INSTALLS: 'false', YARN_ENABLE_TELEMETRY: '0' }
runCommand(binaries.yarn ?? 'yarn', ['install', '--mode', 'update-lockfile'], { cwd: dir, env })
return fs.readFileSync(path.join(dir, 'yarn.lock'), 'utf8')
}
if (manager === 'yarn_classic') {
const nmDir = path.join(dir, '.y1_nm')
const args = ['install', '--ignore-scripts', '--no-node-version-check', '--non-interactive', '--modules-folder', nmDir]
runCommand(binaries.yarnClassic ?? 'yarn', args, { cwd: dir })
return fs.readFileSync(path.join(dir, 'yarn.lock'), 'utf8')
}
throw new Error(`Unknown package manager: ${manager}`)
}

export function generateLockfileForState (manager, state, workDir, binaries = {}) {
if (state.seed) {
generateLockfileForState(manager, state.seed, workDir, binaries)
writeProjectState(workDir, state, { clean: false })
} else {
writeProjectState(workDir, state, { clean: true })
}
if (manager === 'pnpm_v13' || manager === 'pnpm_v9') {
writePnpmWorkspaceYaml(workDir, state, manager === 'pnpm_v13')
}
return invokeManager(manager, workDir, binaries)
}

export function gitMergeLockfiles (baseText, branchAText, branchBText) {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'lockfile-merge-'))
try {
const baseFile = path.join(dir, 'base')
const aFile = path.join(dir, 'a')
const bFile = path.join(dir, 'b')
fs.writeFileSync(baseFile, baseText)
fs.writeFileSync(aFile, branchAText)
fs.writeFileSync(bFile, branchBText)
const result = spawn.sync('git', ['merge-file', '-p', aFile, baseFile, bFile], { encoding: 'utf8' })
if (result.error) throw result.error
if (result.status == null || result.status < 0 || result.status > 127) {
throw new Error(`git merge-file failed with status ${result.status}: ${result.stderr ?? ''}`)
}
const merged = result.stdout ?? ''
const conflicts = countConflictHunks(merged)
return { conflicts, merged }
Comment thread
zkochan marked this conversation as resolved.
Resolved
} finally {
fs.rmSync(dir, { recursive: true, force: true })
}
}

export function countConflictHunks (text) {
let count = 0
for (const line of text.split('\n')) {
if (line.startsWith('<<<<<<<')) count += 1
}
return count
}

export function evaluateScenarioOutcome ({ base, branchA, branchB, expectedMerged }) {
const { conflicts, merged } = gitMergeLockfiles(base, branchA, branchB)
const exactMatch = conflicts === 0 && merged === expectedMerged
const semanticDrift = conflicts === 0 && !exactMatch
return { conflicts, exactMatch, semanticDrift }
}

function summarizeManagerOutcomes (scenarios, manager) {
let clean = 0
let conflicted = 0
let conflictHunks = 0
let semanticDrift = 0
for (const scenario of Object.values(scenarios)) {
const outcome = scenario.managers[manager]
if (!outcome) continue
if (outcome.conflicts > 0) {
conflicted += 1
conflictHunks += outcome.conflicts
} else if (outcome.exactMatch) {
clean += 1
} else {
semanticDrift += 1
}
}
return { clean, conflicted, conflictHunks, semanticDrift }
}

export function summarizeConflictResults (scenarios, managers = CONFLICT_MANAGERS) {
const summary = {}
for (const manager of managers) {
summary[manager] = summarizeManagerOutcomes(scenarios, manager)
}
return summary
}

export function runLockfileConflictsBenchmark ({
managers = CONFLICT_MANAGERS,
binaries = {},
outputPath = OUTPUT_FILE,
} = {}) {
const workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lockfile-bench-work-'))
const scenarios = {}
try {
for (const scenario of lockfileConflictScenarios) {
scenarios[scenario.id] = { description: scenario.description, managers: {} }
for (const manager of managers) {
const base = generateLockfileForState(manager, scenario.base, workDir, binaries)
const branchA = generateLockfileForState(manager, scenario.branchA, workDir, binaries)
const branchB = generateLockfileForState(manager, scenario.branchB, workDir, binaries)
const expectedMerged = generateLockfileForState(manager, scenario.merged, workDir, binaries)
scenarios[scenario.id].managers[manager] = evaluateScenarioOutcome({
base,
branchA,
branchB,
expectedMerged,
})
}
}
} finally {
fs.rmSync(workDir, { recursive: true, force: true })
}
const report = {
measuredAt: new Date().toISOString(),
totalScenarios: lockfileConflictScenarios.length,
summary: summarizeConflictResults(scenarios, managers),
scenarios,
}
if (outputPath) {
fs.writeFileSync(outputPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8')
}
return report
}

if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) {
const report = runLockfileConflictsBenchmark()
console.log(JSON.stringify(report.summary, null, 2))
}
114 changes: 114 additions & 0 deletions benchmarkLockfileConflicts.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
import assert from 'node:assert/strict'
import fs from 'node:fs'
import path from 'node:path'
import test from 'node:test'

import tempy from 'tempy'

import {
CONFLICT_MANAGERS,
countConflictHunks,
evaluateScenarioOutcome,
gitMergeLockfiles,
summarizeConflictResults,
writeProjectState,
} from './benchmarkLockfileConflicts.js'
import { lockfileConflictScenarios } from './lockfileConflictScenarios.js'

test('gitMergeLockfiles merges non-overlapping multi-line dependency maps cleanly', () => {
const base = [
'{',
'\t"projects": {',
'\t\t".": {',
'\t\t\t"dependencies": {',
'\t\t\t\t"fast-copy": {',
'\t\t\t\t\t"specifier": "~3.0.0",',
'\t\t\t\t\t"version": "3.0.0",',
'\t\t\t\t},',
'\t\t\t\t"fast-redact": {',
'\t\t\t\t\t"specifier": "~3.4.0",',
'\t\t\t\t\t"version": "3.4.0",',
'\t\t\t\t},',
'\t\t\t},',
'\t\t},',
'\t},',
'}',
'',
].join('\n')
const branchA = base
.replace('"~3.0.0"', '"^3.1.0"')
.replace('"3.0.0"', '"3.1.0"')
const branchB = base
.replace('"~3.4.0"', '"^3.5.0"')
.replace('"3.4.0"', '"3.5.0"')
const expectedMerged = branchA
.replace('"~3.4.0"', '"^3.5.0"')
.replace('"3.4.0"', '"3.5.0"')

const outcome = evaluateScenarioOutcome({
base,
branchA,
branchB,
expectedMerged,
})
assert.deepEqual(outcome, {
conflicts: 0,
exactMatch: true,
semanticDrift: false,
})
})

test('gitMergeLockfiles counts conflict hunks and detects semantic drift', () => {
const base = 'a: 1\nb: 2\n'
const branchA = 'a: 10\nb: 2\n'
const branchB = 'a: 1\nb: 20\n'
const conflictedMerge = gitMergeLockfiles(base, branchA, branchB)
assert.equal(conflictedMerge.conflicts, 1)
assert.equal(countConflictHunks(conflictedMerge.merged), 1)

const driftOutcome = evaluateScenarioOutcome({
base: 'a: 1\n\nb: 2\n',
branchA: 'a: 10\n\nb: 2\n',
branchB: 'a: 1\n\nb: 20\n',
expectedMerged: 'a: 10\n\nb: 99\n',
})
assert.deepEqual(driftOutcome, {
conflicts: 0,
exactMatch: false,
semanticDrift: true,
})
})

test('summarizeConflictResults aggregates clean, conflicted, and drift scenarios', () => {
const scenarios = {
S01: {
managers: {
pnpm_v13: { conflicts: 0, exactMatch: true, semanticDrift: false },
npm: { conflicts: 2, exactMatch: false, semanticDrift: false },
},
},
S02: {
managers: {
pnpm_v13: { conflicts: 0, exactMatch: false, semanticDrift: true },
npm: { conflicts: 0, exactMatch: true, semanticDrift: false },
},
},
}
const summary = summarizeConflictResults(scenarios, ['pnpm_v13', 'npm'])
assert.deepEqual(summary, {
pnpm_v13: { clean: 1, conflicted: 0, conflictHunks: 0, semanticDrift: 1 },
npm: { clean: 1, conflicted: 1, conflictHunks: 2, semanticDrift: 0 },
})
})

test('writeProjectState writes root and workspace manifests and scenarios are well-formed', () => {
const dir = tempy.directory()
writeProjectState(dir, {
dependencies: { abbrev: '3.0.1' },
workspaces: { a: { dependencies: { ini: '5.0.0' } } },
})
assert.ok(fs.existsSync(path.join(dir, 'package.json')))
assert.ok(fs.existsSync(path.join(dir, 'packages/a/package.json')))
assert.equal(lockfileConflictScenarios.length, 25)
assert.equal(CONFLICT_MANAGERS.includes('pnpm_v13'), true)
})
Loading
Loading