From 6e0dc8c9bc4b56935d95d7676261fc9bc5e47a0a Mon Sep 17 00:00:00 2001 From: Miguel Grau Date: Fri, 7 Aug 2026 08:49:04 +0200 Subject: [PATCH 1/3] Fix processes silently running with cpus=1 and consolidate them into 4 documented cpu/memory tiers. --- conf/base.config | 67 +++++++++++++++---- .../deepcsa_config_bbgregressions_test.config | 15 +++-- conf/mice.config | 5 -- conf/test_real.config | 15 +++-- modules/local/analyzedepths/main.nf | 2 +- .../bbgregressions/create_input/main.nf | 1 - .../bbgregressions/editconfig/main.nf | 2 +- .../bbgtools/bbgregressions/plot/main.nf | 1 - .../bbgregressions/regressions/main.nf | 1 - .../local/bbgtools/omega/estimator/main.nf | 2 - .../local/bbgtools/omega/mutabilities/main.nf | 3 - .../local/bbgtools/oncodrive3d/plot/main.nf | 2 +- .../oncodrive3d/plot_chimerax/main.nf | 2 +- .../oncodrive3d/preprocessing/main.nf | 2 +- .../local/bbgtools/oncodrive3d/run/main.nf | 2 +- .../local/bbgtools/oncodriveclustl/main.nf | 2 +- modules/local/bbgtools/oncodrivefml/main.nf | 2 +- modules/local/bbgtools/sitecomparison/main.nf | 3 - modules/local/blacklistmuts/main.nf | 2 +- modules/local/compute_mutability/main.nf | 3 +- modules/local/compute_profile/main.nf | 3 +- modules/local/compute_trinucleotide/main.nf | 2 +- modules/local/computedepths/main.nf | 3 + modules/local/concatprofiles/main.nf | 3 +- modules/local/contamination/main.nf | 2 +- modules/local/createmaskmatrix/main.nf | 4 +- modules/local/createpanels/compare/main.nf | 1 - .../local/createpanels/custombedfile/main.nf | 3 +- modules/local/dnds/adaptpanelrefcds/main.nf | 4 +- modules/local/dnds/buildref/main.nf | 3 - modules/local/dnds/preprocess/main.nf | 3 - modules/local/dnds/run/main.nf | 3 - modules/local/dnds_proxy/main.nf | 1 - modules/local/downsample/depths/main.nf | 2 +- modules/local/downsample/mutations/main.nf | 2 +- modules/local/expand_regions/main.nf | 2 +- modules/local/filterdepths/main.nf | 1 - modules/local/hotspots_selection/main.nf | 3 - modules/local/indels/main.nf | 1 - modules/local/maf2vcf/main.nf | 2 +- modules/local/mut_density/adjusted/main.nf | 2 +- modules/local/mutated_cells_expected/main.nf | 2 +- modules/local/mutated_cells_from_vaf/main.nf | 2 - .../local/mutated_genomes_from_vaf/main.nf | 2 - modules/local/mutation_matrix/main.nf | 3 +- modules/local/omega_multipletesting/main.nf | 2 +- .../plot/interindividual_variability/main.nf | 2 +- modules/local/plot/needles/main.nf | 1 + modules/local/plot/omega/main.nf | 2 +- modules/local/plot/qc/annotate_omega/main.nf | 2 +- .../plot/qc/globalloc_synonymous/main.nf | 2 +- .../local/plot/qc/metrics_vs_depth/main.nf | 2 +- .../local/plot/qc/mutation_densities/main.nf | 2 +- .../local/plot/qc/mutation_specific/main.nf | 2 +- .../plot/qc/omega_vs_global_vs_dndscv/main.nf | 2 +- modules/local/plot/saturation/main.nf | 2 +- .../local/plot/saturation/proportions/main.nf | 2 +- modules/local/plot/selection_metrics/main.nf | 3 +- .../local/process_annotation/domain/main.nf | 1 + .../local/process_annotation/panel/main.nf | 1 + .../signatures/hdp/chainsampling/main.nf | 2 +- .../local/signatures/hdp/compare_sigs/main.nf | 2 +- .../signatures/hdp/normalize_sigs/main.nf | 2 +- .../local/signatures/hdp/prepareinput/main.nf | 2 +- .../signatures/hdp/process_results/main.nf | 2 +- .../signatures/hdp/reformat_sigs/main.nf | 2 +- modules/local/signatures/msighdp/main.nf | 2 +- modules/local/signatures/musical/main.nf | 1 - .../sigprofiler/assignment/cosmic_fit/main.nf | 2 +- .../assignment/decompose_fit/main.nf | 2 +- .../signatures/sigprofiler/extractor/main.nf | 2 +- .../sigprofiler/matrixgenerator/main.nf | 1 - modules/local/sortpanel/main.nf | 2 + modules/local/subsetmaf/main.nf | 1 + modules/local/table2groups/main.nf | 1 + .../custom/dumpsoftwareversions/main.nf | 3 +- modules/nf-core/ensemblvep/veppanel/main.nf | 2 - modules/nf-core/multiqc/main.nf | 2 + modules/nf-core/tabix/bgziptabix/main.nf | 3 +- 79 files changed, 140 insertions(+), 117 deletions(-) diff --git a/conf/base.config b/conf/base.config index fbacfc64..2b5c57c0 100644 --- a/conf/base.config +++ b/conf/base.config @@ -32,27 +32,69 @@ process { maxRetries = 2 } + // === CPU TIERS === + // Label-based, cpus only. Memory/time stay in the withName overrides + // below so per-process tuning isn't blocked by a shared label. No + // label for cpus=1: that's already the global default above. + withLabel: cpu_low { + cpus = { 2 * task.attempt } + } + withLabel: cpu_medium { + cpus = { 4 * task.attempt } + } + withLabel: cpu_high { + cpus = { 8 * task.attempt } + } + + // === MEMORY TIERS === + // Only one label: processes verified (via peak_rss trace data) to stay + // under 1 GB, where the 4 GB global default is wasteful at scale. + // Anything above 4 GB gets a dedicated withName override instead + // (see e.g. DNDS:PREPROCESSDEPTHS, MUTABILITY.*:RELATIVEMUTABILITY + // below) so tuning stays per-process rather than a shared label. + withLabel: mem_low { + memory = { 1.GB * task.attempt } + } + // === PANEL CREATION PROCESSES === // Large memory requirements for genomic position processing withName: 'BBGTOOLS:DEEPCSA:CREATEPANELS:SITESFROMPOSITIONS' { memory = { 8.GB * task.attempt } } - // VEP annotation is CPU and memory intensive for large VCFs + // VEP annotation is CPU and memory intensive for large VCFs. + // Exception to the cpu-tier labels: verified via real trace data at + // ~1200-1400% cpu (12-14 cores used out of 24 requested) on panel chunks. withName: 'BBGTOOLS:DEEPCSA:CREATEPANELS:VCFANNOTATEPANEL:ENSEMBLVEP_VEP*' { cpus = { 24 * task.attempt } memory = { 6.GB * task.attempt } } + // Per-sample VEP annotation: no label (module is nf-core, unlabeled); + // kept as its own cpus override, same as the panel one above. + withName: 'BBGTOOLS:DEEPCSA:MUT_PREPROCESSING:VCFANNOTATE:ENSEMBLVEP_VEP*' { + cpus = { 24 * task.attempt } + } + withName: 'CUSTOMPROCESSING.*' { memory = { 10.GB * task.attempt } } + // Exception to the cpu-tier labels: TABIX_BGZIPTABIX_QUERY is a shared + // nf-core module reused under many other aliases (QUERYMUTATIONS, + // QUERYDEPTHS, QUERYPANEL...) that don't need extra cpus, so this can't + // be a module-level label without affecting those too. withName: 'BBGTOOLS:DEEPCSA:DEPTHS.*CONS' { cpus = { 2 * task.attempt } memory = { 8.GB * task.attempt } } + // Same shared module, different aliases: these query mutation tables + // (not depths), peak_rss well under 1 GB. + withName: 'QUERYMUTATIONS|QUERYMUTATIONSEXONS' { + memory = { 1.GB * task.attempt } + } + withName: 'BBGTOOLS:DEEPCSA:CREATEPANELS:CREATECAPTUREDPANELS*' { memory = { 6.GB * task.attempt } } @@ -68,7 +110,6 @@ process { } withName: '(BBGTOOLS:DEEPCSA:CREATEPANELS:DOMAINANNOTATION*)' { - cpus = { 2 * task.attempt } memory = { 10.GB * task.attempt } } @@ -84,13 +125,7 @@ process { memory = { 12.GB * task.attempt } } - withName: 'BBGTOOLS:DEEPCSA:MUT_PREPROCESSING:SOMATICMUTATIONS*' { - cpus = { 1 * task.attempt } - memory = { 1.GB * task.attempt } - } - withName: 'BBGTOOLS:DEEPCSA:CREATEPANELS:POSTPROCESSVEPPANEL*' { - cpus = { 2 * task.attempt } memory = { 8.GB * task.attempt } } @@ -104,16 +139,22 @@ process { memory = { 2.GB * task.attempt } } + // peak_rss 33-33.4 GB (BDO_duplexome / all_samples) -> +20% headroom + withName: 'BBGTOOLS:DEEPCSA:DNDS:PREPROCESSDEPTHS' { + memory = { 40.GB * task.attempt } + } + + // peak_rss 36.1-36.9 GB, same order of magnitude on both MUTABILITYALL + // and MUTABILITYNONPROT (BDO_duplexome / all_samples) -> +19% headroom + withName: 'BBGTOOLS:DEEPCSA:MUTABILITY.*:RELATIVEMUTABILITY' { + memory = { 44.GB * task.attempt } + } + // === UTILITY PROCESSES === withName: 'BBGTOOLS:DEEPCSA:CUSTOM_DUMPSOFTWAREVERSIONS*' { cache = false } - withName: 'BBGTOOLS:DEEPCSA:DEPTHANALYSIS:COMPUTEDEPTHS' { - cpus = { 4 * task.attempt } - memory = { 4.GB * task.attempt } - } - withName: 'MUTATEDGENOMESFROMVAFAM' { errorStrategy = 'ignore' maxRetries = 1 diff --git a/conf/deepcsa_config_bbgregressions_test.config b/conf/deepcsa_config_bbgregressions_test.config index a6aa2ee3..99c3dc03 100644 --- a/conf/deepcsa_config_bbgregressions_test.config +++ b/conf/deepcsa_config_bbgregressions_test.config @@ -100,14 +100,17 @@ params { } process { - withLabel: process_high_memory { + withLabel: cpu_high { cpus = { 2 * task.attempt } - memory = { 15.GB * task.attempt } + memory = { 16.GB * task.attempt } } - withLabel: process_high { - cpus = { 2 * task.attempt } - memory = { 16.GB * task.attempt } - time = { 1.h * task.attempt } + // base.config sizes these for real cohorts (40/44 GB); cap them back + // down for the minimal test dataset. + withName: 'BBGTOOLS:DEEPCSA:DNDS:PREPROCESSDEPTHS' { + memory = { 15.GB * task.attempt } + } + withName: 'BBGTOOLS:DEEPCSA:MUTABILITY.*:RELATIVEMUTABILITY' { + memory = { 15.GB * task.attempt } } } diff --git a/conf/mice.config b/conf/mice.config index fa4e3fae..ded884c1 100644 --- a/conf/mice.config +++ b/conf/mice.config @@ -77,8 +77,3 @@ params { } -process { - withLabel: process_high_memory { - memory = { 15.GB * task.attempt } - } -} diff --git a/conf/test_real.config b/conf/test_real.config index 8cc9ed7f..b97f5385 100644 --- a/conf/test_real.config +++ b/conf/test_real.config @@ -97,14 +97,17 @@ params { } process { - withLabel: process_high_memory { + withLabel: cpu_high { cpus = { 2 * task.attempt } - memory = { 15.GB * task.attempt } + memory = { 16.GB * task.attempt } } - withLabel: process_high { - cpus = { 2 * task.attempt } - memory = { 16.GB * task.attempt } - time = { 1.h * task.attempt } + // base.config sizes these for real cohorts (40/44 GB); cap them back + // down for the minimal test dataset. + withName: 'BBGTOOLS:DEEPCSA:DNDS:PREPROCESSDEPTHS' { + memory = { 15.GB * task.attempt } + } + withName: 'BBGTOOLS:DEEPCSA:MUTABILITY.*:RELATIVEMUTABILITY' { + memory = { 15.GB * task.attempt } } } diff --git a/modules/local/analyzedepths/main.nf b/modules/local/analyzedepths/main.nf index d4efb13e..4366742f 100644 --- a/modules/local/analyzedepths/main.nf +++ b/modules/local/analyzedepths/main.nf @@ -1,6 +1,6 @@ process ANALYZE_DEPTHS_GROUPS { tag "groups" - label 'process_low' + label 'cpu_low' label 'deepcsa_core' diff --git a/modules/local/bbgtools/bbgregressions/create_input/main.nf b/modules/local/bbgtools/bbgregressions/create_input/main.nf index ee3f79db..d6b8d2c9 100644 --- a/modules/local/bbgtools/bbgregressions/create_input/main.nf +++ b/modules/local/bbgtools/bbgregressions/create_input/main.nf @@ -1,7 +1,6 @@ process CREATE_INPUT { tag "regressions" - label 'process_single' label "bbgregressions" diff --git a/modules/local/bbgtools/bbgregressions/editconfig/main.nf b/modules/local/bbgtools/bbgregressions/editconfig/main.nf index b1ca5e14..e33d8e82 100644 --- a/modules/local/bbgtools/bbgregressions/editconfig/main.nf +++ b/modules/local/bbgtools/bbgregressions/editconfig/main.nf @@ -1,7 +1,7 @@ process EDITCONFIG { tag "regressions" - label 'process_low' + label 'cpu_low' label "bbgregressions" diff --git a/modules/local/bbgtools/bbgregressions/plot/main.nf b/modules/local/bbgtools/bbgregressions/plot/main.nf index 6d8687a6..420d2513 100644 --- a/modules/local/bbgtools/bbgregressions/plot/main.nf +++ b/modules/local/bbgtools/bbgregressions/plot/main.nf @@ -1,7 +1,6 @@ process PLOT { tag "regressions" - label 'process_single' label "bbgregressions" diff --git a/modules/local/bbgtools/bbgregressions/regressions/main.nf b/modules/local/bbgtools/bbgregressions/regressions/main.nf index 2f45cf0c..feed85b4 100644 --- a/modules/local/bbgtools/bbgregressions/regressions/main.nf +++ b/modules/local/bbgtools/bbgregressions/regressions/main.nf @@ -1,7 +1,6 @@ process REGRESSIONS { tag "regressions" - label 'process_single' label "bbgregressions" diff --git a/modules/local/bbgtools/omega/estimator/main.nf b/modules/local/bbgtools/omega/estimator/main.nf index 44ebf696..edcb668c 100644 --- a/modules/local/bbgtools/omega/estimator/main.nf +++ b/modules/local/bbgtools/omega/estimator/main.nf @@ -1,8 +1,6 @@ process OMEGA_ESTIMATOR { tag "$meta.id" label 'cpu_medium' - label 'time_low' - label 'process_high_memory' container 'docker.io/bbglab/omega:0.2.1' diff --git a/modules/local/bbgtools/omega/mutabilities/main.nf b/modules/local/bbgtools/omega/mutabilities/main.nf index 765327db..d4fab89c 100644 --- a/modules/local/bbgtools/omega/mutabilities/main.nf +++ b/modules/local/bbgtools/omega/mutabilities/main.nf @@ -1,8 +1,5 @@ process OMEGA_MUTABILITIES { tag "$meta.id" - label 'cpu_single_fixed' - label 'time_low' - label 'process_high_memory' container 'docker.io/bbglab/omega:0.2.1' diff --git a/modules/local/bbgtools/oncodrive3d/plot/main.nf b/modules/local/bbgtools/oncodrive3d/plot/main.nf index ee8d79e3..c3c0a4c7 100644 --- a/modules/local/bbgtools/oncodrive3d/plot/main.nf +++ b/modules/local/bbgtools/oncodrive3d/plot/main.nf @@ -1,6 +1,6 @@ process ONCODRIVE3D_PLOT { tag "$meta.id" - label 'process_medium' + label 'cpu_medium' container 'docker.io/spellegrini87/oncodrive3d:1.0.9-light' diff --git a/modules/local/bbgtools/oncodrive3d/plot_chimerax/main.nf b/modules/local/bbgtools/oncodrive3d/plot_chimerax/main.nf index bdc24664..5a5d968a 100644 --- a/modules/local/bbgtools/oncodrive3d/plot_chimerax/main.nf +++ b/modules/local/bbgtools/oncodrive3d/plot_chimerax/main.nf @@ -1,6 +1,6 @@ process ONCODRIVE3D_PLOT_CHIMERAX { tag "$meta.id" - label 'process_medium' + label 'cpu_medium' container 'docker.io/spellegrini87/oncodrive3d:1.0.9-chimerax' diff --git a/modules/local/bbgtools/oncodrive3d/preprocessing/main.nf b/modules/local/bbgtools/oncodrive3d/preprocessing/main.nf index c769e5e9..c43637ba 100644 --- a/modules/local/bbgtools/oncodrive3d/preprocessing/main.nf +++ b/modules/local/bbgtools/oncodrive3d/preprocessing/main.nf @@ -1,6 +1,6 @@ process ONCODRIVE3D_PREPROCESSING { tag "$meta.id" - label 'process_medium' + label 'cpu_medium' label 'deepcsa_core' diff --git a/modules/local/bbgtools/oncodrive3d/run/main.nf b/modules/local/bbgtools/oncodrive3d/run/main.nf index 43752d2e..6aa07487 100644 --- a/modules/local/bbgtools/oncodrive3d/run/main.nf +++ b/modules/local/bbgtools/oncodrive3d/run/main.nf @@ -1,6 +1,6 @@ process ONCODRIVE3D_RUN { tag "$meta.id" - label 'process_high' + label 'cpu_high' container 'docker.io/spellegrini87/oncodrive3d:1.0.9-light' diff --git a/modules/local/bbgtools/oncodriveclustl/main.nf b/modules/local/bbgtools/oncodriveclustl/main.nf index 51476280..a4e205d8 100644 --- a/modules/local/bbgtools/oncodriveclustl/main.nf +++ b/modules/local/bbgtools/oncodriveclustl/main.nf @@ -1,6 +1,6 @@ process ONCODRIVECLUSTL { tag "$meta.id" - label 'process_high' + label 'cpu_high' // // conda "YOUR-TOOL-HERE" // container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? diff --git a/modules/local/bbgtools/oncodrivefml/main.nf b/modules/local/bbgtools/oncodrivefml/main.nf index bea1acee..f882ec88 100644 --- a/modules/local/bbgtools/oncodrivefml/main.nf +++ b/modules/local/bbgtools/oncodrivefml/main.nf @@ -1,6 +1,6 @@ process ONCODRIVEFML { tag "$meta.id" - label 'process_high' + label 'cpu_high' container 'docker.io/ferriolcalvet/oncodrivefml:latest' diff --git a/modules/local/bbgtools/sitecomparison/main.nf b/modules/local/bbgtools/sitecomparison/main.nf index 5813002d..7c3dd60c 100644 --- a/modules/local/bbgtools/sitecomparison/main.nf +++ b/modules/local/bbgtools/sitecomparison/main.nf @@ -1,8 +1,5 @@ process SITE_COMPARISON { tag "$meta.id" - label 'cpu_single_fixed' - label 'time_low' - label 'process_high_memory' label 'deepcsa_core' diff --git a/modules/local/blacklistmuts/main.nf b/modules/local/blacklistmuts/main.nf index fe940ad8..69bac7d0 100644 --- a/modules/local/blacklistmuts/main.nf +++ b/modules/local/blacklistmuts/main.nf @@ -1,7 +1,7 @@ process BLACKLIST_MUTATIONS { tag "$meta.id" - label 'process_low' + label 'cpu_low' label 'deepcsa_core' diff --git a/modules/local/compute_mutability/main.nf b/modules/local/compute_mutability/main.nf index ce5e8804..ab37ea8f 100644 --- a/modules/local/compute_mutability/main.nf +++ b/modules/local/compute_mutability/main.nf @@ -1,8 +1,7 @@ process COMPUTE_RELATIVE_MUTABILITY { tag "$meta.id" - label 'process_low_fixed_cpus' - label 'process_high_memory' + label 'cpu_low' label 'deepcsa_core' diff --git a/modules/local/compute_profile/main.nf b/modules/local/compute_profile/main.nf index 89039194..1e529147 100644 --- a/modules/local/compute_profile/main.nf +++ b/modules/local/compute_profile/main.nf @@ -1,7 +1,8 @@ process COMPUTE_PROFILE { tag "$meta.id" - label 'process_low' + label 'cpu_low' + label 'mem_low' label 'deepcsa_core' diff --git a/modules/local/compute_trinucleotide/main.nf b/modules/local/compute_trinucleotide/main.nf index f356e267..e0d46213 100644 --- a/modules/local/compute_trinucleotide/main.nf +++ b/modules/local/compute_trinucleotide/main.nf @@ -1,7 +1,7 @@ process COMPUTE_TRINUCLEOTIDE { tag "$meta.id" - label 'process_low' + label 'cpu_low' label 'deepcsa_core' diff --git a/modules/local/computedepths/main.nf b/modules/local/computedepths/main.nf index 6279a636..c40d121a 100644 --- a/modules/local/computedepths/main.nf +++ b/modules/local/computedepths/main.nf @@ -1,6 +1,9 @@ process COMPUTEDEPTHS { tag "$meta.id" + label 'cpu_medium' + label 'mem_low' + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/samtools:1.18--h50ea8bc_1' : diff --git a/modules/local/concatprofiles/main.nf b/modules/local/concatprofiles/main.nf index 3cb6c5f8..3d4f6fad 100644 --- a/modules/local/concatprofiles/main.nf +++ b/modules/local/concatprofiles/main.nf @@ -1,6 +1,7 @@ process CONCAT_PROFILES { tag "${meta.id}" - label 'process_medium' + label 'cpu_medium' + label 'mem_low' label 'deepcsa_core' input: diff --git a/modules/local/contamination/main.nf b/modules/local/contamination/main.nf index 84dc1de0..5bcc346e 100644 --- a/modules/local/contamination/main.nf +++ b/modules/local/contamination/main.nf @@ -1,7 +1,7 @@ process COMPUTE_CONTAMINATION { tag "$meta.id" - label 'process_high' + label 'cpu_high' label 'deepcsa_core' diff --git a/modules/local/createmaskmatrix/main.nf b/modules/local/createmaskmatrix/main.nf index e703c218..2a796369 100644 --- a/modules/local/createmaskmatrix/main.nf +++ b/modules/local/createmaskmatrix/main.nf @@ -1,8 +1,8 @@ process CREATE_MASK_MATRIX { tag "mask_matrix" - label 'process_low' - label 'time_low' + label 'cpu_low' + label 'mem_low' container "docker.io/bbglab/deepcsa-core:0.1.0" diff --git a/modules/local/createpanels/compare/main.nf b/modules/local/createpanels/compare/main.nf index 12e90350..20691f00 100644 --- a/modules/local/createpanels/compare/main.nf +++ b/modules/local/createpanels/compare/main.nf @@ -1,6 +1,5 @@ process COMPARE_TRINUCLEOTIDE_PROPORTIONS_PANELS { tag "1" - label 'process_single' label 'deepcsa_core' diff --git a/modules/local/createpanels/custombedfile/main.nf b/modules/local/createpanels/custombedfile/main.nf index c39f9b2b..2e10a98e 100644 --- a/modules/local/createpanels/custombedfile/main.nf +++ b/modules/local/createpanels/custombedfile/main.nf @@ -1,6 +1,7 @@ process CREATECUSTOMBEDFILE { tag "$meta.id" - label 'process_single' + + label 'mem_low' conda "bioconda::pybedtools=0.9.1--py38he0f268d_0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? diff --git a/modules/local/dnds/adaptpanelrefcds/main.nf b/modules/local/dnds/adaptpanelrefcds/main.nf index 3e3972c1..f8e382b8 100644 --- a/modules/local/dnds/adaptpanelrefcds/main.nf +++ b/modules/local/dnds/adaptpanelrefcds/main.nf @@ -1,9 +1,7 @@ process ADAPT_PANEL_REFCDS { tag "$meta.id" - label 'cpu_single_fixed' - label 'time_low' - label 'process_high_memory' + label 'mem_low' label 'deepcsa_core' diff --git a/modules/local/dnds/buildref/main.nf b/modules/local/dnds/buildref/main.nf index bc38b99e..de3c9a3e 100644 --- a/modules/local/dnds/buildref/main.nf +++ b/modules/local/dnds/buildref/main.nf @@ -1,9 +1,6 @@ process BUILD_REFCDS { tag "$meta.id" - label 'cpu_single_fixed' - label 'time_low' - label 'process_high_memory' container 'docker.io/ferriolcalvet/dnds:latest' diff --git a/modules/local/dnds/preprocess/main.nf b/modules/local/dnds/preprocess/main.nf index 050cf7c4..c87e2bca 100644 --- a/modules/local/dnds/preprocess/main.nf +++ b/modules/local/dnds/preprocess/main.nf @@ -1,8 +1,5 @@ process PREPROCESS_DNDS { tag "$meta.id" - label 'cpu_single_fixed' - label 'time_low' - label 'process_high_memory' label 'deepcsa_core' diff --git a/modules/local/dnds/run/main.nf b/modules/local/dnds/run/main.nf index 5ea84dfb..9addf262 100644 --- a/modules/local/dnds/run/main.nf +++ b/modules/local/dnds/run/main.nf @@ -1,8 +1,5 @@ process RUN_DNDS { tag "$meta.id" - label 'cpu_single_fixed' - label 'time_low' - label 'process_high_memory' container 'docker.io/ferriolcalvet/dnds:latest' diff --git a/modules/local/dnds_proxy/main.nf b/modules/local/dnds_proxy/main.nf index 5179ad4a..23a878a1 100644 --- a/modules/local/dnds_proxy/main.nf +++ b/modules/local/dnds_proxy/main.nf @@ -1,6 +1,5 @@ process DNDS_PROXY { tag "$meta.id" - label 'process_single' label 'deepcsa_core' diff --git a/modules/local/downsample/depths/main.nf b/modules/local/downsample/depths/main.nf index f8dd14f4..9996093e 100644 --- a/modules/local/downsample/depths/main.nf +++ b/modules/local/downsample/depths/main.nf @@ -1,7 +1,7 @@ process DOWNSAMPLE_DEPTHS { tag "$meta.id" - label 'process_high' + label 'cpu_high' label 'deepcsa_core' diff --git a/modules/local/downsample/mutations/main.nf b/modules/local/downsample/mutations/main.nf index d28c4e20..3452d388 100644 --- a/modules/local/downsample/mutations/main.nf +++ b/modules/local/downsample/mutations/main.nf @@ -4,7 +4,7 @@ process DOWNSAMPLE_MUTATIONS { cache false tag "$meta.id" - label 'process_high' + label 'cpu_high' label 'deepcsa_core' diff --git a/modules/local/expand_regions/main.nf b/modules/local/expand_regions/main.nf index b4ca7097..8550d5f2 100644 --- a/modules/local/expand_regions/main.nf +++ b/modules/local/expand_regions/main.nf @@ -1,7 +1,7 @@ process EXPAND_REGIONS { tag "$meta.id" - label 'process_high' + label 'cpu_high' label 'deepcsa_core' diff --git a/modules/local/filterdepths/main.nf b/modules/local/filterdepths/main.nf index 22867592..139d7b82 100644 --- a/modules/local/filterdepths/main.nf +++ b/modules/local/filterdepths/main.nf @@ -1,6 +1,5 @@ process FILTERDEPTHS { tag "$meta.id" - label 'process_single' label 'deepcsa_core' diff --git a/modules/local/hotspots_selection/main.nf b/modules/local/hotspots_selection/main.nf index 31ce3e3f..6c0444e5 100644 --- a/modules/local/hotspots_selection/main.nf +++ b/modules/local/hotspots_selection/main.nf @@ -1,8 +1,5 @@ process HOTSPOTS_SELECTION { tag "$meta.id" - label 'cpu_single_fixed' - label 'time_low' - label 'process_high_memory' label 'deepcsa_core' diff --git a/modules/local/indels/main.nf b/modules/local/indels/main.nf index baf0aadd..ac17a56a 100644 --- a/modules/local/indels/main.nf +++ b/modules/local/indels/main.nf @@ -1,6 +1,5 @@ process INDELS_COMPARISON { tag "$meta.id" - label 'process_single' label 'deepcsa_core' diff --git a/modules/local/maf2vcf/main.nf b/modules/local/maf2vcf/main.nf index 51c32e1f..6d079efd 100644 --- a/modules/local/maf2vcf/main.nf +++ b/modules/local/maf2vcf/main.nf @@ -1,7 +1,7 @@ process MAF_2_VCF { tag "${meta.id}" - label 'process_low' + label 'cpu_low' label 'deepcsa_core' diff --git a/modules/local/mut_density/adjusted/main.nf b/modules/local/mut_density/adjusted/main.nf index eb855631..c92d6b0e 100644 --- a/modules/local/mut_density/adjusted/main.nf +++ b/modules/local/mut_density/adjusted/main.nf @@ -1,7 +1,7 @@ process MUTATION_DENSITY { tag "$meta.id" - label 'process_high' + label 'cpu_high' label 'deepcsa_core' diff --git a/modules/local/mutated_cells_expected/main.nf b/modules/local/mutated_cells_expected/main.nf index e9f5ad93..c9fdc40c 100644 --- a/modules/local/mutated_cells_expected/main.nf +++ b/modules/local/mutated_cells_expected/main.nf @@ -1,6 +1,6 @@ process EXPECTED_MUTATED_CELLS { tag "$meta.id" - label 'process_medium' + label 'cpu_medium' container 'docker.io/axelrosendahlhuber/expected_mutrate:latest' diff --git a/modules/local/mutated_cells_from_vaf/main.nf b/modules/local/mutated_cells_from_vaf/main.nf index d24bd31c..04b78cfa 100644 --- a/modules/local/mutated_cells_from_vaf/main.nf +++ b/modules/local/mutated_cells_from_vaf/main.nf @@ -1,7 +1,5 @@ process MUTATED_CELLS_FROM_VAF { tag "$meta.id" - label 'process_single' - label 'memory_medium' container 'docker.io/ferranmuinos/test_mutated_genomes' diff --git a/modules/local/mutated_genomes_from_vaf/main.nf b/modules/local/mutated_genomes_from_vaf/main.nf index e9a5644d..eafdc0f5 100644 --- a/modules/local/mutated_genomes_from_vaf/main.nf +++ b/modules/local/mutated_genomes_from_vaf/main.nf @@ -1,7 +1,5 @@ process MUTATED_GENOMES_FROM_VAF { tag "$meta.id" - label 'process_single' - label 'memory_medium' container 'docker.io/ferranmuinos/test_mutated_genomes' diff --git a/modules/local/mutation_matrix/main.nf b/modules/local/mutation_matrix/main.nf index 924dffae..fa4f43f8 100644 --- a/modules/local/mutation_matrix/main.nf +++ b/modules/local/mutation_matrix/main.nf @@ -1,7 +1,8 @@ process COMPUTE_MATRIX { tag "$meta.id" - label 'process_low' + label 'cpu_low' + label 'mem_low' label 'deepcsa_core' diff --git a/modules/local/omega_multipletesting/main.nf b/modules/local/omega_multipletesting/main.nf index 2a0b9397..9cd94900 100644 --- a/modules/local/omega_multipletesting/main.nf +++ b/modules/local/omega_multipletesting/main.nf @@ -1,6 +1,6 @@ process OMEGA_MULTITEST { tag "all" - label 'process_low' + label 'cpu_low' label 'deepcsa_core' diff --git a/modules/local/plot/interindividual_variability/main.nf b/modules/local/plot/interindividual_variability/main.nf index 50c83031..602957c7 100644 --- a/modules/local/plot/interindividual_variability/main.nf +++ b/modules/local/plot/interindividual_variability/main.nf @@ -1,7 +1,7 @@ process PLOT_INTERINDIVIDUAL_VARIABILITY { tag "samples" - label 'process_low' + label 'cpu_low' label 'deepcsa_core' diff --git a/modules/local/plot/needles/main.nf b/modules/local/plot/needles/main.nf index bd0fbe90..d77ded50 100644 --- a/modules/local/plot/needles/main.nf +++ b/modules/local/plot/needles/main.nf @@ -3,6 +3,7 @@ process PLOT_NEEDLES { tag "$meta.id" label 'deepcsa_core' + label 'mem_low' input: tuple val(meta), path(mut_files) diff --git a/modules/local/plot/omega/main.nf b/modules/local/plot/omega/main.nf index 4d5f03ac..0b4de9d1 100644 --- a/modules/local/plot/omega/main.nf +++ b/modules/local/plot/omega/main.nf @@ -1,7 +1,7 @@ process PLOT_OMEGA { tag "$meta.id" - label 'process_low' + label 'cpu_low' label 'deepcsa_core' diff --git a/modules/local/plot/qc/annotate_omega/main.nf b/modules/local/plot/qc/annotate_omega/main.nf index d3a60270..e469aeb4 100644 --- a/modules/local/plot/qc/annotate_omega/main.nf +++ b/modules/local/plot/qc/annotate_omega/main.nf @@ -1,7 +1,7 @@ process ANNOTATE_OMEGA_QC { tag "all_samples" - label 'process_low' + label 'cpu_low' label 'deepcsa_core' diff --git a/modules/local/plot/qc/globalloc_synonymous/main.nf b/modules/local/plot/qc/globalloc_synonymous/main.nf index 6530ca76..cdc6bd15 100644 --- a/modules/local/plot/qc/globalloc_synonymous/main.nf +++ b/modules/local/plot/qc/globalloc_synonymous/main.nf @@ -1,7 +1,7 @@ process PLOT_OMEGASYN_QC { tag "all" - label 'process_low' + label 'cpu_low' label 'deepcsa_core' diff --git a/modules/local/plot/qc/metrics_vs_depth/main.nf b/modules/local/plot/qc/metrics_vs_depth/main.nf index 89229935..a315d7bb 100644 --- a/modules/local/plot/qc/metrics_vs_depth/main.nf +++ b/modules/local/plot/qc/metrics_vs_depth/main.nf @@ -1,7 +1,7 @@ process PLOT_METRICS_VS_DEPTH_QC { tag "${group_name}" - label 'process_low' + label 'cpu_low' label 'deepcsa_core' diff --git a/modules/local/plot/qc/mutation_densities/main.nf b/modules/local/plot/qc/mutation_densities/main.nf index 45cfa07f..74ed4349 100644 --- a/modules/local/plot/qc/mutation_densities/main.nf +++ b/modules/local/plot/qc/mutation_densities/main.nf @@ -1,7 +1,7 @@ process PLOT_MUTDENSITY_QC { tag "${group_name}" - label 'process_low' + label 'cpu_low' label 'deepcsa_core' diff --git a/modules/local/plot/qc/mutation_specific/main.nf b/modules/local/plot/qc/mutation_specific/main.nf index d2cf97a1..8394e122 100644 --- a/modules/local/plot/qc/mutation_specific/main.nf +++ b/modules/local/plot/qc/mutation_specific/main.nf @@ -1,7 +1,7 @@ process PLOT_MUTATION_SPECIFIC { tag "${meta.id}" - label 'process_low' + label 'cpu_low' label 'deepcsa_core' diff --git a/modules/local/plot/qc/omega_vs_global_vs_dndscv/main.nf b/modules/local/plot/qc/omega_vs_global_vs_dndscv/main.nf index 59cfede1..932deccb 100644 --- a/modules/local/plot/qc/omega_vs_global_vs_dndscv/main.nf +++ b/modules/local/plot/qc/omega_vs_global_vs_dndscv/main.nf @@ -1,7 +1,7 @@ process PLOT_OMEGA_VS_GLOBAL_VS_DNDSCV { tag "all_samples" - label 'process_low' + label 'cpu_low' label 'deepcsa_core' input: diff --git a/modules/local/plot/saturation/main.nf b/modules/local/plot/saturation/main.nf index f974cb11..62dbe37f 100644 --- a/modules/local/plot/saturation/main.nf +++ b/modules/local/plot/saturation/main.nf @@ -1,7 +1,7 @@ process PLOT_SATURATION { tag "$meta.id" - label 'process_low' + label 'cpu_low' label 'deepcsa_core' diff --git a/modules/local/plot/saturation/proportions/main.nf b/modules/local/plot/saturation/proportions/main.nf index cd4c1750..e7d1803f 100644 --- a/modules/local/plot/saturation/proportions/main.nf +++ b/modules/local/plot/saturation/proportions/main.nf @@ -1,7 +1,7 @@ process PLOT_SATURATION_PROPORTIONS { tag "$meta.id" - label 'process_low' + label 'cpu_low' label 'deepcsa_core' diff --git a/modules/local/plot/selection_metrics/main.nf b/modules/local/plot/selection_metrics/main.nf index 723a0617..93dc6d70 100644 --- a/modules/local/plot/selection_metrics/main.nf +++ b/modules/local/plot/selection_metrics/main.nf @@ -1,7 +1,8 @@ process PLOT_SELECTION_METRICS { tag "$meta.id" - label 'process_low' + label 'cpu_low' + label 'mem_low' label 'deepcsa_core' diff --git a/modules/local/process_annotation/domain/main.nf b/modules/local/process_annotation/domain/main.nf index 0645dcfc..af1add6d 100644 --- a/modules/local/process_annotation/domain/main.nf +++ b/modules/local/process_annotation/domain/main.nf @@ -3,6 +3,7 @@ process DOMAIN_ANNOTATION { tag "${meta.id}" label 'deepcsa_core' + label 'cpu_low' input: tuple val(meta) , path(panel_annotated) diff --git a/modules/local/process_annotation/panel/main.nf b/modules/local/process_annotation/panel/main.nf index 29f97a06..1b4565a8 100644 --- a/modules/local/process_annotation/panel/main.nf +++ b/modules/local/process_annotation/panel/main.nf @@ -3,6 +3,7 @@ process POSTPROCESS_VEP_ANNOTATION { tag "${meta.id}" label 'deepcsa_core' + label 'cpu_low' input: diff --git a/modules/local/signatures/hdp/chainsampling/main.nf b/modules/local/signatures/hdp/chainsampling/main.nf index 9236bc2c..dadd4d0c 100644 --- a/modules/local/signatures/hdp/chainsampling/main.nf +++ b/modules/local/signatures/hdp/chainsampling/main.nf @@ -1,6 +1,6 @@ process RUN_HDP_CHAIN_SAMPLING { tag "$meta.id" - label 'process_medium' + label 'cpu_medium' container 'docker.io/ferriolcalvet/hdp_wrapper' diff --git a/modules/local/signatures/hdp/compare_sigs/main.nf b/modules/local/signatures/hdp/compare_sigs/main.nf index 1ce062f5..4d0995b5 100644 --- a/modules/local/signatures/hdp/compare_sigs/main.nf +++ b/modules/local/signatures/hdp/compare_sigs/main.nf @@ -1,7 +1,7 @@ process COMPARE_SIGNATURES { tag "$meta.id" - label 'process_medium' + label 'cpu_medium' container 'docker.io/ferriolcalvet/hdp_wrapper' diff --git a/modules/local/signatures/hdp/normalize_sigs/main.nf b/modules/local/signatures/hdp/normalize_sigs/main.nf index a5c52428..03db5657 100644 --- a/modules/local/signatures/hdp/normalize_sigs/main.nf +++ b/modules/local/signatures/hdp/normalize_sigs/main.nf @@ -1,7 +1,7 @@ process NORMALIZE_SIGNATURES { tag "$meta.id" - label 'process_medium' + label 'cpu_medium' container 'docker.io/ferriolcalvet/hdp_wrapper' diff --git a/modules/local/signatures/hdp/prepareinput/main.nf b/modules/local/signatures/hdp/prepareinput/main.nf index 94a6470d..3a5b5396 100644 --- a/modules/local/signatures/hdp/prepareinput/main.nf +++ b/modules/local/signatures/hdp/prepareinput/main.nf @@ -2,7 +2,7 @@ process PREPARE_INPUT { tag "$meta.id" - label 'process_medium' + label 'cpu_medium' container 'docker.io/ferriolcalvet/hdp_wrapper' diff --git a/modules/local/signatures/hdp/process_results/main.nf b/modules/local/signatures/hdp/process_results/main.nf index 9ab6d67b..87dc504c 100644 --- a/modules/local/signatures/hdp/process_results/main.nf +++ b/modules/local/signatures/hdp/process_results/main.nf @@ -1,6 +1,6 @@ process PROCESS_HDP_RESULTS { tag "$meta.id" - label 'process_medium' + label 'cpu_medium' container 'docker.io/ferriolcalvet/hdp_wrapper' diff --git a/modules/local/signatures/hdp/reformat_sigs/main.nf b/modules/local/signatures/hdp/reformat_sigs/main.nf index 0087967b..e98ba36b 100644 --- a/modules/local/signatures/hdp/reformat_sigs/main.nf +++ b/modules/local/signatures/hdp/reformat_sigs/main.nf @@ -1,7 +1,7 @@ process SIGNATURES_HDP_TO_SIGPROFILER { tag "$meta.id" - label 'process_medium' + label 'cpu_medium' label 'deepcsa_core' diff --git a/modules/local/signatures/msighdp/main.nf b/modules/local/signatures/msighdp/main.nf index cef51345..737c66ce 100644 --- a/modules/local/signatures/msighdp/main.nf +++ b/modules/local/signatures/msighdp/main.nf @@ -5,7 +5,7 @@ process MSIGHDP { tag "$meta.id" - label 'process_medium' + label 'cpu_medium' container 'docker.io/ferriolcalvet/msighdp:latest' diff --git a/modules/local/signatures/musical/main.nf b/modules/local/signatures/musical/main.nf index 6d720485..d4fcd1f9 100644 --- a/modules/local/signatures/musical/main.nf +++ b/modules/local/signatures/musical/main.nf @@ -4,7 +4,6 @@ process MUSICAL { tag "$meta.id" - label 'process_high' label 'cpu_high' container 'docker.io/ferriolcalvet/musical:latest' diff --git a/modules/local/signatures/sigprofiler/assignment/cosmic_fit/main.nf b/modules/local/signatures/sigprofiler/assignment/cosmic_fit/main.nf index 8eb9f243..575d3733 100644 --- a/modules/local/signatures/sigprofiler/assignment/cosmic_fit/main.nf +++ b/modules/local/signatures/sigprofiler/assignment/cosmic_fit/main.nf @@ -1,6 +1,6 @@ process SIGPROFILERASSIGNMENT_COSMIC_FIT { tag "$meta.id" - label 'process_medium' + label 'cpu_medium' container 'docker.io/ferriolcalvet/sigprofiler_assignment:1.1.3' diff --git a/modules/local/signatures/sigprofiler/assignment/decompose_fit/main.nf b/modules/local/signatures/sigprofiler/assignment/decompose_fit/main.nf index 1cbc07e8..5a46c315 100644 --- a/modules/local/signatures/sigprofiler/assignment/decompose_fit/main.nf +++ b/modules/local/signatures/sigprofiler/assignment/decompose_fit/main.nf @@ -1,6 +1,6 @@ process SIGPROFILERASSIGNMENT_DECOMPOSE_FIT { tag "$meta.id" - label 'process_medium' + label 'cpu_medium' container 'docker.io/ferriolcalvet/sigprofiler_assignment:1.1.3' diff --git a/modules/local/signatures/sigprofiler/extractor/main.nf b/modules/local/signatures/sigprofiler/extractor/main.nf index 4f774066..afac7072 100644 --- a/modules/local/signatures/sigprofiler/extractor/main.nf +++ b/modules/local/signatures/sigprofiler/extractor/main.nf @@ -5,7 +5,7 @@ process SIGPROFILEREXTRACTOR { tag "$meta.id" - label 'process_medium' + label 'cpu_medium' container 'docker.io/ferriolcalvet/sigprofilerassignment' diff --git a/modules/local/signatures/sigprofiler/matrixgenerator/main.nf b/modules/local/signatures/sigprofiler/matrixgenerator/main.nf index eee651a9..c5db7657 100644 --- a/modules/local/signatures/sigprofiler/matrixgenerator/main.nf +++ b/modules/local/signatures/sigprofiler/matrixgenerator/main.nf @@ -1,6 +1,5 @@ process SIGPROFILER_MATRIXGENERATOR { tag "samples" - label 'process_single' container 'docker.io/ferriolcalvet/sigprofilermatrixgenerator:1.3.5' diff --git a/modules/local/sortpanel/main.nf b/modules/local/sortpanel/main.nf index e7dc683f..87af6817 100644 --- a/modules/local/sortpanel/main.nf +++ b/modules/local/sortpanel/main.nf @@ -2,6 +2,8 @@ process SORT_MERGED_PANEL { tag "${meta.id}" + label 'mem_low' + container "docker.io/bbglab/deepcsa-core:0.0.2-alpha" input: diff --git a/modules/local/subsetmaf/main.nf b/modules/local/subsetmaf/main.nf index ed9467ef..1f206c41 100644 --- a/modules/local/subsetmaf/main.nf +++ b/modules/local/subsetmaf/main.nf @@ -3,6 +3,7 @@ process SUBSET_MAF { tag "$meta.id" label 'deepcsa_core' + label 'mem_low' input: tuple val(meta), path(mut_files) diff --git a/modules/local/table2groups/main.nf b/modules/local/table2groups/main.nf index 8a8159a4..846415cc 100644 --- a/modules/local/table2groups/main.nf +++ b/modules/local/table2groups/main.nf @@ -2,6 +2,7 @@ process TABLE_2_GROUP { tag "groups" label 'deepcsa_core' + label 'mem_low' input: path(features_table) diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf index 2984da18..adf54454 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -1,5 +1,6 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { - label 'process_single' + + label 'mem_low' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container conda "bioconda::multiqc=1.18" diff --git a/modules/nf-core/ensemblvep/veppanel/main.nf b/modules/nf-core/ensemblvep/veppanel/main.nf index eea60f95..dcbbed76 100644 --- a/modules/nf-core/ensemblvep/veppanel/main.nf +++ b/modules/nf-core/ensemblvep/veppanel/main.nf @@ -2,8 +2,6 @@ process ENSEMBLVEP_VEP { tag "$meta.id" - label 'process_high' - label 'cpu_veryhigh' conda params.vep_cache_version == 108 ? 'bioconda::ensembl-vep=108.2' : params.vep_cache_version == 102 ? 'bioconda::ensembl-vep=102.0' : diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index ee0f3097..175dc4ed 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -1,5 +1,7 @@ process MULTIQC { + label 'mem_low' + conda "bioconda::multiqc=1.20" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/multiqc:1.20--pyhdfd78af_0' : diff --git a/modules/nf-core/tabix/bgziptabix/main.nf b/modules/nf-core/tabix/bgziptabix/main.nf index 815657a9..3cec9b49 100644 --- a/modules/nf-core/tabix/bgziptabix/main.nf +++ b/modules/nf-core/tabix/bgziptabix/main.nf @@ -3,7 +3,8 @@ process TABIX_BGZIPTABIX { // cache false tag "$meta.id" - label 'process_single' + + label 'mem_low' conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? From 5894081ae7dc5c1c0ce990e49986c230f4b91269 Mon Sep 17 00:00:00 2001 From: Miguel Grau Date: Thu, 13 Aug 2026 16:25:19 +0200 Subject: [PATCH 2/3] Fix under-provisioned memory tiers based on trace data and add ignore/retry error strategies for flaky processes --- conf/base.config | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/conf/base.config b/conf/base.config index 2b5c57c0..6ce84f9f 100644 --- a/conf/base.config +++ b/conf/base.config @@ -129,11 +129,19 @@ process { memory = { 8.GB * task.attempt } } - // BBGTOOLS:DEEPCSA:OMEGA:ESTIMATOR and BBGTOOLS:DEEPCSA:OMEGA:PREPROCESSING - withName: 'PREPROCESSING|ESTIMATOR' { + // BBGTOOLS:DEEPCSA:OMEGA:PREPROCESSING(GLOBALLOC): peak_rss ~2.6 GB, well under + // the 4 GB default. + withName: 'PREPROCESSING' { memory = { 4.GB * task.attempt } } + // BBGTOOLS:DEEPCSA:{OMEGA,OMEGAMULTI}:{ESTIMATOR,ESTIMATORGLOBALLOC}: peak_rss up + // to ~24.3 GB on the largest gene sets. The old 4 GB tier maxed out at 16 GB after + // 3 retries, so the biggest tasks could never succeed. +20% headroom. + withName: 'ESTIMATOR|ESTIMATORGLOBALLOC' { + memory = { 30.GB * task.attempt } + } + // BBGTOOLS:DEEPCSA:SIGNATURESNONPROT:SIGPROFILERASSIGNMENT withName: 'SIGPROFILERASSIGNMENT' { memory = { 2.GB * task.attempt } @@ -150,6 +158,26 @@ process { memory = { 44.GB * task.attempt } } + // peak_rss up to 39.4 GB on the largest cohorts; no override existed before, so it + // defaulted to 4 GB and could never succeed (63% FAILED in resources_test run). +20% headroom. + withName: 'BBGTOOLS:DEEPCSA:ONCODRIVE3D:ONCODRIVE3D_RUN' { + memory = { 48.GB * task.attempt } + } + + // peak_rss up to 12 GB; no override existed, so every sample needed ~2 retries + // against the 4 GB default before landing on enough memory. +25% headroom so it + // succeeds on the first attempt. + withName: 'BBGTOOLS:DEEPCSA:MUT_PREPROCESSING:FILTERNANOSEQNOISE' { + memory = { 15.GB * task.attempt } + } + + // peak_rss up to 6.3 GB; the 4 GB default meant every one of the 128 samples + // failed on the first attempt and only succeeded on the 8 GB retry. Setting the + // base to 8 GB avoids that guaranteed-fail-then-retry pattern. + withName: 'BBGTOOLS:DEEPCSA:MUTDENSITYALL:MUTDENSITY' { + memory = { 8.GB * task.attempt } + } + // === UTILITY PROCESSES === withName: 'BBGTOOLS:DEEPCSA:CUSTOM_DUMPSOFTWAREVERSIONS*' { cache = false @@ -165,12 +193,18 @@ process { maxRetries = 1 } + withName: 'BBGTOOLS:DEEPCSA:PLOTTINGSUMMARY:PLOTSATURATION' { + errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' } + maxRetries = 4 + } + withName: 'BBGTOOLS:DEEPCSA:PLOTTINGQC:PLOTMUTATIONSPECIFIC' { memory = { 24.GB * task.attempt } } withName: 'BBGTOOLS:DEEPCSA:ENRICHPANELS:DNA2PROTEINMAPPING' { memory = { 30.GB * task.attempt } + errorStrategy = 'ignore' } withName: 'FILTERNANOSEQSNP' { From 653b527d72badd55282655023663d8eca8762777 Mon Sep 17 00:00:00 2001 From: Miguel Grau Date: Fri, 14 Aug 2026 22:53:01 +0200 Subject: [PATCH 3/3] Fix under-provisioned memory for ONCODRIVEFMLSNVS cohort runs, decoupling cpus from attempt-scaling so retries gain real headroom --- conf/base.config | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/conf/base.config b/conf/base.config index 6ce84f9f..9566d183 100644 --- a/conf/base.config +++ b/conf/base.config @@ -178,6 +178,19 @@ process { memory = { 8.GB * task.attempt } } + // Same process handles per-sample and per-cohort ("all_samples", "Ismale0/1", + // "Samplelocation*") runs -- can't select by tag, only by process name. The + // cohort runs were getting OOM-killed by Slurm (71-113 oom_kill events per job) + // under the 4 GB default; individual samples fit comfortably under 3 GB. No clean + // peak_rss from the killed runs to calibrate against, so 24 GB is an estimate -- + // revisit once a run confirms the real peak. cpus fixed (not cpu_high's + // 8*attempt) so retries actually gain memory headroom per core instead of just + // scaling both in lockstep. + withName: 'BBGTOOLS:DEEPCSA:ONCODRIVEFMLALL:ONCODRIVEFMLSNVS' { + cpus = { 8 } + memory = { 24.GB * task.attempt } + } + // === UTILITY PROCESSES === withName: 'BBGTOOLS:DEEPCSA:CUSTOM_DUMPSOFTWAREVERSIONS*' { cache = false