diff --git a/docs/config.mdx b/docs/config.mdx index 0d80f0e361..e7caf4d0b0 100644 --- a/docs/config.mdx +++ b/docs/config.mdx @@ -284,6 +284,38 @@ This configuration: The same selector syntax and the same priority order apply in the `agent` scope, which configures agent tasks independently from the `process` scope. See [Agents](agent.mdx#configuration). +### Disabling processes + +You can use the `when` setting to disable a process at runtime. When `when` is `false`, the process does not execute any tasks: + +```groovy +process { + withName: 'FASTQC' { + when = false + } +} +``` + +The `when` setting can also be a closure, which is evaluated for each task. The closure can reference task inputs and params: + +```groovy +process { + withName: 'FASTQC' { + when = { meta.id != params.skip_sample } + } +} +``` + +The `when` setting overrides the `when` section of a process, if it has one. + +A disabled process emits no outputs, so downstream processes that depend on it will not execute either. + +The `when` setting is intended for temporarily disabling a process. Conditional logic that is part of the pipeline should be implemented in the calling workflow (e.g., using an `if` statement or [`filter`][operator-filter] operator). + +:::warning +Disabling a process that emits dataflow values (i.e., all of its inputs are dataflow values) produces empty dataflow values, which will likely cause the pipeline to fail or hang. Only disable processes that emit channels. +::: + ## Config profiles Configuration files can define one or more *profiles*. A profile is a set of configuration settings that can be selected at runtime using the `-profile` command line option. @@ -348,6 +380,7 @@ This approach is useful for handling workflow events without modifying the pipel [cli-params]: ./cli#pipeline-parameters [config-options]: ./reference/config [nxf-env-vars]: ./reference/env-vars#nextflow-settings +[operator-filter]: ./reference/operator#filter [process-reference]: ./reference/process [process-label]: ./reference/process/directives/label [secrets-page]: ./secrets diff --git a/docs/deprecations.mdx b/docs/deprecations.mdx index db9a580489..123e5aa577 100644 --- a/docs/deprecations.mdx +++ b/docs/deprecations.mdx @@ -23,7 +23,6 @@ See [Migration notes][migrations-page] for the full list of changes in each rele | Workflow `\|` and `&` operators | Method calls and assignments | | `.out` to access process/workflow outputs | Assignments | | [Legacy operators][operator-page] | [Core operators][operator-typed-page] | -| [Process `when` section][strict-process-when] | Conditional logic in the calling workflow | | Using global variables in a process | Process inputs | | [`publishDir`][process-publishDir] directive | [Workflow outputs][workflow-outputs] | @@ -41,11 +40,7 @@ See [Migration notes][migrations-page] for the full list of changes in each rele | `Channel` to access channel factories | 25.10 | [`channel`][strict-channel] | | Legacy syntax parser (`NXF_SYNTAX_PARSER=v1`) | 26.04 | [Strict syntax][strict-syntax-page] | | `channel.fromSRA()` | 26.04 | [Entrez Direct](https://www.ncbi.nlm.nih.gov/books/NBK179288/) | - -### Process directives - -| Feature | Deprecated | Replacement | -|---|---|---| +| [Process `when` section][strict-process-when] | 26.10 | Conditional logic in the calling workflow, or [`process.when`][config-process-when] config setting | | [`storeDir`][process-storeDir] | 26.10 | [Explicit workflow logic][process-storeDir-alt] | ### Configuration @@ -91,6 +86,7 @@ See [Migration notes][migrations-page] for the full list of changes in each rele | `echo` directive | 22.04 | 26.10 | `debug` | [config-manifest]: ./reference/config/manifest +[config-process-when]: ./config#disabling-processes [config-seqera-executor-autoLabels]: ./reference/config/seqera#seqeraexecutorautolabels [config-tower-autoLabels]: ./reference/config/tower#towerautolabels [config-workflow-handlers]: ./config#workflow-handlers diff --git a/docs/migrations/26-10.mdx b/docs/migrations/26-10.mdx index a0db08eb53..a81c8e1ae1 100644 --- a/docs/migrations/26-10.mdx +++ b/docs/migrations/26-10.mdx @@ -222,6 +222,8 @@ The hash of `eval` commands has changed. This fixes a bug where tasks with `eval ## Deprecations +- The process `when` section is deprecated. Use explicit workflow logic, or use the [`process.when`][config-process-when] config setting to disable a process at runtime. See [Process when section][strict-syntax-process-when] for more information. + - The `storeDir` process directive is deprecated. Use explicit workflow logic to reuse an intermediate output if it is present and compute it otherwise. See [`storeDir`][process-storeDir] for an example. - The [`seqera.executor.autoLabels`][config-seqera-executor-autoLabels] config option is deprecated. Use [`tower.autoLabels`][config-tower-autoLabels] instead, which applies to every executor that supports the `resourceLabels` directive. @@ -252,6 +254,7 @@ The hash of `eval` commands has changed. This fixes a bug where tasks with `eval [config-dag-directory]: ../reference/config/dag#dagdirectory [config-docker-cpuLimits]: ../reference/config/docker#dockercpulimits [config-k8s-computeResourceType]: ../reference/config/k8s#k8scomputeresourcetype +[config-process-when]: ../config#disabling-processes [config-manifest-diagram]: ../reference/config/manifest#manifestdiagram [config-report-directory]: ../reference/config/report#reportdirectory [config-seqera-executor-autoLabels]: ../reference/config/seqera#seqeraexecutorautolabels @@ -269,5 +272,6 @@ The hash of `eval` commands has changed. This fixes a bug where tasks with `eval [plugin-registry]: ../plugins/plugin-registry#configuring-plugin-registries [process-storeDir]: ../reference/process/directives/store-dir [static-typing]: ../static-typing +[strict-syntax-process-when]: ../strict-syntax#process-when-section [workflow-output-def]: ../workflow#outputs [workflow-typed-params]: ../typed-parameters diff --git a/docs/process.mdx b/docs/process.mdx index 018ce02ab4..40fb717a8b 100644 --- a/docs/process.mdx +++ b/docs/process.mdx @@ -1117,9 +1117,9 @@ See also: [Multiple inputs][process-multiple-inputs]. ## When -:::note -As a best practice, conditional logic should be implemented in the calling workflow (e.g. using an `if` statement or [`filter`][operator-filter] operator) instead of the process definition. -::: + +Implement conditional logic in the calling workflow (e.g. an `if` statement or [`filter`][operator-filter] operator), or use the [`process.when`][config-process-when] config setting to disable a process at runtime. + The `when` section allows you to define a condition that must be satisfied in order to execute the process. The condition can be any expression that returns a boolean value. @@ -1320,6 +1320,7 @@ process hello { [cache-nondeterministic-inputs]: ./cache-and-resume#non-deterministic-process-inputs [channel-value]: ./reference/stdlib-namespaces/channel#value +[config-process-when]: ./config#disabling-processes [executor-page]: ./executor [glob]: http://docs.oracle.com/javase/tutorial/essential/io/fileOps.html.mdx#glob [operator-combine]: ./reference/operator#combine diff --git a/docs/strict-syntax.mdx b/docs/strict-syntax.mdx index cc4295c1f0..470f9a9eb7 100644 --- a/docs/strict-syntax.mdx +++ b/docs/strict-syntax.mdx @@ -529,6 +529,35 @@ ch.map { v -> v * 2 } // correct ch.map { it -> it * 2 } // also correct ``` +### Process when section + +The process [when][process-when] section is deprecated. Implement conditional logic in the calling workflow (e.g. an `if` statement or [`filter`][operator-filter] operator) instead. + +To disable a process at runtime, use the [`process.when`][config-process-when] config setting instead. For example: + +```nextflow +process fastqc { + // ... + + when: + task.ext.when == null || task.ext.when + + // ... +} +``` + +The `when:` section can be removed, and any config settings for `ext.when` can be replaced with `when`: + +```groovy +process { + withName: 'FASTQC' { + when = { !params.skip_fastqc } + } +} +``` + +The `process.when` config setting takes precedence over the `when` section. You can therefore move from `ext.when` to `process.when` in your config before the `when` section is removed from the process. + ### Process shell section The process `shell` section is deprecated. Use the `script` section instead. The strict parser provides error checking to help distinguish between Nextflow variables and Bash variables. @@ -621,10 +650,6 @@ workflow { } ``` -### Process when section - -The process [when][process-when] section is discouraged. As a best practice, conditional logic should be implemented in the calling workflow (e.g. using an `if` statement or [filter][operator-filter] operator) instead of the process definition. - ## Configuration syntax See [Configuration][config-syntax] for a comprehensive description of the configuration language. @@ -708,6 +733,7 @@ Any Groovy code can be moved into the `lib` directory, which supports the full G For Groovy code that is complicated or if it depends on third-party libraries, it may be better to create a plugin. Plugins can define custom functions that can be included by Nextflow scripts like a script definition. Furthermore, plugins can be easily reused across different pipelines. See [Developing plugins][plugins-dev-page] for more information on how to develop plugins. +[config-process-when]: ./config#disabling-processes [config-syntax]: ./config#syntax [dsl1-page]: ./migrations/dsl1 [lib-directory]: ./sharing#the-lib-directory diff --git a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessConfigBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessConfigBuilder.groovy index 90da6a2ade..7490fd702f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessConfigBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/dsl/ProcessConfigBuilder.groovy @@ -177,7 +177,7 @@ class ProcessConfigBuilder extends ProcessBuilder { if( entry.key in ignoredKeys ) // e.g. the agent-only options of the `agent` scope continue - if( !DIRECTIVES.contains(entry.key) ) + if( !DIRECTIVES.contains(entry.key) && entry.key != 'when' ) log.warn "Unknown directive `$entry.key` for $kind `$processName`" if( entry.key == 'params' ) // <-- patch issue #242 diff --git a/modules/nextflow/src/test/groovy/nextflow/script/ScriptRunnerTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/ScriptRunnerTest.groovy index a9e87339cf..40d85dec4d 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/ScriptRunnerTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/ScriptRunnerTest.groovy @@ -23,6 +23,7 @@ import nextflow.processor.TaskProcessor import nextflow.util.Duration import nextflow.util.MemoryUnit import spock.lang.Timeout +import spock.lang.Unroll import test.Dsl2Spec import static test.ScriptHelper.* @@ -635,4 +636,39 @@ class ScriptRunnerTest extends Dsl2Spec { result.val == "echo foo" } + + @Unroll + def 'should disable a process with process.when config setting' () { + given: + def config = loadConfig(CONFIG) + + def script = ''' + process hola { + input: + val x + + output: + stdout + + script: + "echo $x" + } + + workflow { + hola(channel.of('a', 'b', 'c')).toList() + } + ''' + + when: + def result = runScript(script, config: config) + + then: + result.val.sort() == EXPECTED + + where: + CONFIG | EXPECTED + "process.executor = 'nope'" | ['echo a', 'echo b', 'echo c'] + "process.executor = 'nope'; process.when = false" | [] + "process { executor = 'nope'; withName: hola { when = { x != 'b' } } }" | ['echo a', 'echo c'] + } } diff --git a/modules/nf-lang/src/main/java/nextflow/config/spec/SpecNode.java b/modules/nf-lang/src/main/java/nextflow/config/spec/SpecNode.java index 5c463db138..2d4e0bd0b9 100644 --- a/modules/nf-lang/src/main/java/nextflow/config/spec/SpecNode.java +++ b/modules/nf-lang/src/main/java/nextflow/config/spec/SpecNode.java @@ -49,11 +49,17 @@ private static Scope rootScope() { // derive `process` and `agent` config options from process directives: an agent // task accepts the same task directives, in its own independent scope. // NOTE: the descriptions are inlined because an interface cannot declare private fields. - result.children().put("process", directiveScope(""" + var processScope = directiveScope(""" The `process` scope allows you to specify default directives for processes in your pipeline. [Read more](https://docs.seqera.io/nextflow/config#process-configuration) - """)); + """); + processScope.children().put("when", new Option(""" + When `false`, the process is not executed. Can be a closure to evaluate the condition for each task. + + [Read more](https://docs.seqera.io/nextflow/config#disabling-processes) + """, List.of(Boolean.class))); + result.children().put("process", processScope); result.children().put("agent", directiveScope(""" The `agent` scope allows you to specify default directives for agents in your pipeline. An agent task accepts the same task directives as a process, in its own independent scope. @@ -98,7 +104,7 @@ else if( fqName.startsWith("nextflow.preview.") ) * * @param description */ - private static SpecNode directiveScope(String description) { + private static Scope directiveScope(String description) { var children = new HashMap(); for( var method : ProcessDsl.DirectiveDsl.class.getDeclaredMethods() ) { if( method.getParameters().length != 1 ) diff --git a/modules/nf-lang/src/main/java/nextflow/script/control/TypeCheckingVisitor.java b/modules/nf-lang/src/main/java/nextflow/script/control/TypeCheckingVisitor.java index 33c9579e26..86032bce50 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/control/TypeCheckingVisitor.java +++ b/modules/nf-lang/src/main/java/nextflow/script/control/TypeCheckingVisitor.java @@ -292,8 +292,6 @@ private void checkWorkflowEmitTypes(Statement block) { public void visitProcessV2(ProcessNodeV2 node) { visitProcessDirectives(node.directives); visit(node.stagers); - if( !(node.when instanceof EmptyExpression) ) - addSoftError("Process `when` section is discouraged with static typing -- use conditional logic in the calling workflow instead", node.when); visit(node.when); visit(node.exec); visit(node.stub); diff --git a/modules/nf-lang/src/main/java/nextflow/script/control/VariableScopeVisitor.java b/modules/nf-lang/src/main/java/nextflow/script/control/VariableScopeVisitor.java index 0c6c799ea9..0679759f1f 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/control/VariableScopeVisitor.java +++ b/modules/nf-lang/src/main/java/nextflow/script/control/VariableScopeVisitor.java @@ -61,7 +61,6 @@ import org.codehaus.groovy.ast.expr.ClosureExpression; import org.codehaus.groovy.ast.expr.ConstantExpression; import org.codehaus.groovy.ast.expr.DeclarationExpression; -import org.codehaus.groovy.ast.expr.EmptyExpression; import org.codehaus.groovy.ast.expr.Expression; import org.codehaus.groovy.ast.expr.MapEntryExpression; import org.codehaus.groovy.ast.expr.MethodCallExpression; @@ -443,8 +442,6 @@ public void visitProcessV1(ProcessNodeV1 node) { visitDirectives(node.inputs, "process input qualifier", false); vsc.popScope(); - if( !(node.when instanceof EmptyExpression) ) - vsc.addParanoidWarning("Process `when` section will not be supported in a future version", node.when); visit(node.when); visit(node.exec); diff --git a/modules/nf-lang/src/main/java/nextflow/script/parser/ScriptAstBuilder.java b/modules/nf-lang/src/main/java/nextflow/script/parser/ScriptAstBuilder.java index 53b1683bd9..f84b545a05 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/parser/ScriptAstBuilder.java +++ b/modules/nf-lang/src/main/java/nextflow/script/parser/ScriptAstBuilder.java @@ -821,6 +821,7 @@ private boolean isDirectiveWithNegativeValue(Expression expression) { private Expression processWhen(ProcessWhenContext ctx) { if( ctx == null ) return EmptyExpression.INSTANCE; + collectWarning("The `when` section is deprecated -- use conditional logic in the calling workflow, or the `process.when` config option to disable a process at runtime", ctx.WHEN().getText(), ast( new EmptyStatement(), ctx.WHEN() )); return ast( expression(ctx.expression()), ctx ); } diff --git a/modules/nf-lang/src/test/groovy/nextflow/script/control/ScriptResolveTest.groovy b/modules/nf-lang/src/test/groovy/nextflow/script/control/ScriptResolveTest.groovy index 70830779dd..7594c72bb8 100644 --- a/modules/nf-lang/src/test/groovy/nextflow/script/control/ScriptResolveTest.groovy +++ b/modules/nf-lang/src/test/groovy/nextflow/script/control/ScriptResolveTest.groovy @@ -267,6 +267,30 @@ class ScriptResolveTest extends Specification { errors[1].getOriginalMessage() == '`y` is not defined' } + def 'should warn about a process `when` section' () { + given: + def source = '''\ + nextflow.enable.types = true + + process hello { + when: + task.ext.when + + exec: + println 'hello!' + } + ''' + when: + def result = scriptParser.parse('main.nf', source.stripIndent()) + scriptParser.analyze() + def errors = TestUtils.getErrors(result) + def warnings = TestUtils.getWarnings(result) + then: + errors.size() == 0 + warnings.size() == 1 + warnings[0].contains('The `when` section is deprecated') + } + def 'should not warn when a workflow emits a channel by name' () { given: def source = '''\ diff --git a/modules/nf-lang/src/test/groovy/nextflow/script/types/TypeCheckingTest.groovy b/modules/nf-lang/src/test/groovy/nextflow/script/types/TypeCheckingTest.groovy index 8fdf2a56f3..3f60a4cd36 100644 --- a/modules/nf-lang/src/test/groovy/nextflow/script/types/TypeCheckingTest.groovy +++ b/modules/nf-lang/src/test/groovy/nextflow/script/types/TypeCheckingTest.groovy @@ -103,29 +103,6 @@ class TypeCheckingTest extends Specification { return true } - def 'should warn about a process `when` section' () { - when: - def errors = getErrors( - '''\ - nextflow.enable.types = true - - process hello { - when: - task.ext.when - - exec: - println 'hello!' - } - ''' - ) - then: - errors.size() == 1 - errors[0].getStartLine() == 4 - errors[0].getStartColumn() == 5 - errors[0].isSoftError() - errors[0].getOriginalMessage() == "Process `when` section is discouraged with static typing -- use conditional logic in the calling workflow instead" - } - @Unroll def 'should report legacy type annotations' () { expect: