From 11430471d5cfe02a5b289b79a6c22cf6ab94896d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Le=20Meur?= Date: Sat, 15 Aug 2026 22:05:52 +0200 Subject: [PATCH 1/2] chore(pipeline): adjustments before balanced splits --- CONTRIBUTING.md | 13 ++++++- Jenkinsfile | 97 +++++++++++++++++++++++++++++++------------------ 2 files changed, 72 insertions(+), 38 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8774fc66a..98bdcde16 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -170,8 +170,17 @@ To run a build against a specific set of repositories and their plugin(s), you c ```diff -def limitedPluginSet = [] +def limitedPluginSet = [ -+ 'jenkinsci/cron_column-plugin\tcron_column', -+ 'jenkinsci/pipeline-stage-view-plugin\tpipeline-rest-api,pipeline-stage-view' ++ 'jenkinsci/aws-credentials-plugin aws-credentials', ++ 'jenkinsci/aws-global-configuration-plugin aws-global-configuration', ++ 'jenkinsci/azure-credentials-plugin azure-credentials', ++ 'jenkinsci/azure-keyvault-plugin azure-keyvault', ++ 'jenkinsci/azure-sdk-plugin azure-sdk', ++ 'jenkinsci/azure-storage-plugin windows-azure-storage', ++ 'jenkinsci/badge-plugin badge', ++ 'jenkinsci/basic-branch-build-strategies-plugin basic-branch-build-strategies', ++ 'jenkinsci/coverage-plugin coverage', ++ 'jenkinsci/cron_column-plugin cron_column', ++ 'jenkinsci/pipeline-maven-plugin pipeline-maven,pipeline-maven-api,pipeline-maven-database', +] ``` diff --git a/Jenkinsfile b/Jenkinsfile index 612a15616..ee2f3d87b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -26,6 +26,7 @@ def mavenEnv(Map params = [:], Closure body) { def attempt = 0 def attempts = 6 retry(count: attempts, conditions: [kubernetesAgent(handleNonKubernetes: true), nonresumable()]) { + def provisioningStart = System.currentTimeMillis() echo 'Attempt ' + ++attempt + ' of ' + attempts // no Dockerized tests; https://github.com/jenkins-infra/documentation/blob/master/ci.adoc#container-agents node('maven-bom') { @@ -36,6 +37,8 @@ def mavenEnv(Map params = [:], Closure body) { 'PATH+JDK=/opt/jdk-' + params['jdk'] + '/bin', "MAVEN_ARGS=${env.MAVEN_ARGS != null ? MAVEN_ARGS : ''} -B ${env.MAVEN_NTP != null ? '-ntp' : ''} -Dmaven.repo.local=${WORKSPACE_TMP}/m2repo", "MVN_LOCAL_REPO=${WORKSPACE_TMP}/m2repo", + "PROVISONING_START=${provisioningStart}", + "CURRENT_ATTEMPT=${attempt}", ]) { infra.loadMavenLocalCacheIfAny(env.MVN_LOCAL_REPO) @@ -62,15 +65,17 @@ def lines def fullTestMarkerFile def weeklyTestMarkerFile def consumeIncrementalsMarkerFile -boolean fullTest = false -boolean weeklyTest = false -boolean consumeIncrementals = false +def fullTest = false +def weeklyTest = false +def consumeIncrementals = false def splits = [:] def results = [:] +def commit +def pctDuration mavenEnv(jdk: 21) { stage('prep') { - checkout scm + commit = checkout(scm).GIT_COMMIT.take(7) consumeIncrementalsMarkerFile = fileExists 'consume-incrementals' consumeIncrementals = consumeIncrementalsMarkerFile || (env.CHANGE_ID && pullRequest.labels.contains('consume-incrementals')) if (!consumeIncrementals) { @@ -109,21 +114,19 @@ mavenEnv(jdk: 21) { lines = ['weekly'] } echo "${pluginsByRepository.size()} repositories:\n${plugins.join('\n')}" - echo "${lines.size()} lines: ${lines.join(' ')} " + echo "${lines.size()} lines: ${lines.join(' ')}" // Fixed splits, each split using only one line lines.each { line -> pluginsByRepository.eachWithIndex { repository, repoPlugins, idx -> def index = (idx % maxSplitsPerLine) + 1 // to get split1 to split def name = "split-${index}:${line}" - def repositoryAndPlugins = "${repository} ${repoPlugins}" - splits[name] = splits[name] ?: [] - splits[name] << repositoryAndPlugins + splits[name] << repository } } - echo "${splits.size()} splits" - echo splits.collect { split, combinations -> "${split} ${combinations}" }.join('\n') + echo "${splits.size()} split(s)" + echo splits.collect { split, repositories -> "${split} (${repositories.size()}) ${repositories}" }.join('\n') } stage('stash line(s)') { lines.each { line -> @@ -134,29 +137,31 @@ mavenEnv(jdk: 21) { if (BRANCH_NAME == 'master' || fullTest || weeklyTest) { stage('run pct') { + def pctStart = System.currentTimeMillis() def branches = [failFast: false] - splits.each { split, combinations -> + splits.each { split, repositories -> def line = split.split(':')[1] def jdk = line == 'weekly' || line == '2.555.x' ? 21 : 17 - branches[split] = { + branches["${split} (${repositories.size()})"] = { + echo "In this split: ${repositories.join(',')}" mavenEnv(jdk: jdk) { + def provisionStart = env.PROVISONING_START.toLong() + def readyIn = (System.currentTimeMillis() - provisionStart) / 1000.0 + echo "INFO: agent ready to run pct in ${readyIn}s" stage('unstash line') { unstash line } - combinations.eachWithIndex { repositoryAndPlugins, idx -> - def parts = repositoryAndPlugins.split(' ') - def repository = parts[0] - def plugins = parts[1] + repositories.eachWithIndex { repository, idx -> def combination = "${repository}:${line}" // if the tests ran with success in a previous attempt, skip the combination (ex: in case of reclaimed spot agent) def previousResult = results[combination] ?: [totalCount: 0, failCount: 0] if (previousResult.totalCount > 0 && previousResult.failCount == 0) { echo "${combination} has already ran ${previousResult.totalCount} test(s) with success in a previous attempt, skipping" } else { - stage("${combination} (${idx + 1}/${combinations.size()})") { + stage("${combination} (${idx + 1}/${repositories.size()})") { withChecks(name: "PCT / ${combination}") { withEnv([ - "PLUGINS=${plugins}", + "PLUGINS=${pluginsByRepository[repository]}", "LINE=$line", "CONSUME_INCREMENTALS=${consumeIncrementals}", 'EXTRA_MAVEN_PROPERTIES=maven.test.failure.ignore=true:surefire.rerunFailingTestsCount=1' @@ -181,37 +186,57 @@ if (BRANCH_NAME == 'master' || fullTest || weeklyTest) { 'elapsed': (System.currentTimeMillis() - start) / 1000.0, 'totalCount': junitResults ? junitResults.totalCount : 0, 'failCount': junitResults ? junitResults.failCount : 0, + 'split': split, + 'readyIn': readyIn, + 'attempt': env.CURRENT_ATTEMPT, ] } } } } + echo "${combination}: ${results[combination]['totalCount']} tests executed in ${results[combination]['elapsed']}s" } } + def totalTime = (System.currentTimeMillis() - provisionStart) / 1000.0 + def runningTime = totalTime - readyIn + echo "INFO: pct tests of ${split} took ${runningTime}s" } } } parallel branches + pctDuration = (System.currentTimeMillis() - pctStart) / 1000.0 + echo "INFO: pct tests took ${pctDuration}s in total" } - stage('duration report') { - node('maven-bom') { - Double totalTime = 0 - def reportLines = '' - results.each { combination, result -> - totalTime += result.elapsed as Double - def normalizedCombination = combination.replace(':', '-') - reportLines += '\n' - } - if (reportLines) { - def content = """ - - ${reportLines} - - """ - writeFile file: 'bom-report.xml', text: content - archiveArtifacts artifacts: 'bom-report.xml' - junit allowEmptyResults: true, testResults: 'bom-report.xml' + node('maven-bom') { + stage('reports') { + def branches = [:] + lines.each { line -> + branches[line] = { + // We need junit records in distinct stages later on for splitTests + // Otherwise it would try to balance all repositories across all lines + // While we want one line per split (agent) + stage("report ${line}") { + def testSuiteName = "bom-report_${line}" + def testCases = [] + results.each { combination, result -> + def repository = combination.split(':')[0] + def resultLine = combination.split(':')[1] + if (line == resultLine) { + testCases << '\n' + } + } + def content = """ + + ${testCases.sort().join('\n')} + + """ + writeFile file: "${testSuiteName}.xml", text: content + junit testResults: "${testSuiteName}.xml" + archiveArtifacts artifacts: "${testSuiteName}.xml" + } + } } + parallel branches } } } From b61a850c3eac98789c08e6d80854090b69953d30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Le=20Meur?= Date: Mon, 17 Aug 2026 01:45:22 +0200 Subject: [PATCH 2/2] adjustments --- CONTRIBUTING.md | 29 ++++++++++++++--------------- Jenkinsfile | 42 ++++++++++++++++++++---------------------- 2 files changed, 34 insertions(+), 37 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 98bdcde16..0dcc56ed5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -167,21 +167,20 @@ It is unusual but possible for cross-component incompatibilities to only be visi ### Running a limited plugin set To run a build against a specific set of repositories and their plugin(s), you can alter the Jenkinsfile via a commit or a replay (if you have permission to do so): -```diff --def limitedPluginSet = [] -+def limitedPluginSet = [ -+ 'jenkinsci/aws-credentials-plugin aws-credentials', -+ 'jenkinsci/aws-global-configuration-plugin aws-global-configuration', -+ 'jenkinsci/azure-credentials-plugin azure-credentials', -+ 'jenkinsci/azure-keyvault-plugin azure-keyvault', -+ 'jenkinsci/azure-sdk-plugin azure-sdk', -+ 'jenkinsci/azure-storage-plugin windows-azure-storage', -+ 'jenkinsci/badge-plugin badge', -+ 'jenkinsci/basic-branch-build-strategies-plugin basic-branch-build-strategies', -+ 'jenkinsci/coverage-plugin coverage', -+ 'jenkinsci/cron_column-plugin cron_column', -+ 'jenkinsci/pipeline-maven-plugin pipeline-maven,pipeline-maven-api,pipeline-maven-database', -+] +```groovy +def limitedPluginSet = [ + 'jenkinsci/aws-credentials-plugin aws-credentials', + 'jenkinsci/aws-global-configuration-plugin aws-global-configuration', + 'jenkinsci/azure-credentials-plugin azure-credentials', + 'jenkinsci/azure-keyvault-plugin azure-keyvault', + 'jenkinsci/azure-sdk-plugin azure-sdk', + 'jenkinsci/azure-storage-plugin windows-azure-storage', + 'jenkinsci/badge-plugin badge', + 'jenkinsci/basic-branch-build-strategies-plugin basic-branch-build-strategies', + 'jenkinsci/coverage-plugin coverage', + 'jenkinsci/cron_column-plugin cron_column', + 'jenkinsci/pipeline-maven-plugin pipeline-maven,pipeline-maven-api,pipeline-maven-database', +] ``` Expected line format: jenkinsci/, tab, diff --git a/Jenkinsfile b/Jenkinsfile index ee2f3d87b..d65fd8f91 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -142,7 +142,7 @@ if (BRANCH_NAME == 'master' || fullTest || weeklyTest) { splits.each { split, repositories -> def line = split.split(':')[1] def jdk = line == 'weekly' || line == '2.555.x' ? 21 : 17 - branches["${split} (${repositories.size()})"] = { + branches["${split} [${repositories.size()}]"] = { echo "In this split: ${repositories.join(',')}" mavenEnv(jdk: jdk) { def provisionStart = env.PROVISONING_START.toLong() @@ -211,29 +211,27 @@ if (BRANCH_NAME == 'master' || fullTest || weeklyTest) { stage('reports') { def branches = [:] lines.each { line -> - branches[line] = { - // We need junit records in distinct stages later on for splitTests - // Otherwise it would try to balance all repositories across all lines - // While we want one line per split (agent) - stage("report ${line}") { - def testSuiteName = "bom-report_${line}" - def testCases = [] - results.each { combination, result -> - def repository = combination.split(':')[0] - def resultLine = combination.split(':')[1] - if (line == resultLine) { - testCases << '\n' - } + def testSuiteName = "bom-report_${line}" + // We need junit records in distinct stages later on for splitTests + // Otherwise it would try to balance all repositories across all lines + // While we want one line per split (agent) + branches[testSuiteName] = { + def testCases = [] + results.each { combination, result -> + def repository = combination.split(':')[0] + def resultLine = combination.split(':')[1] + if (line == resultLine) { + testCases << '\n' } - def content = """ - - ${testCases.sort().join('\n')} - - """ - writeFile file: "${testSuiteName}.xml", text: content - junit testResults: "${testSuiteName}.xml" - archiveArtifacts artifacts: "${testSuiteName}.xml" } + def content = """ + + ${testCases.sort().join('\n')} + + """ + writeFile file: "${testSuiteName}.xml", text: content + junit testResults: "${testSuiteName}.xml" + archiveArtifacts artifacts: "${testSuiteName}.xml" } } parallel branches