diff --git a/CHANGELOG.md b/CHANGELOG.md index caf4789f..62cf44e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,9 @@ Thank you to everyone else that has contributed by reporting bugs, enhancements - [PR #382](https://github.com/nf-core/fetchngs/pull/382) - Use csv.reader to fix stray quote and comma handling in multiqc_mappings_config (#375) - [PR #383](https://github.com/nf-core/fetchngs/pull/383) - Add ampliseq pipeline support - [PR #384](https://github.com/nf-core/fetchngs/pull/384) - Add mag pipeline support +- [PR #385](https://github.com/nf-core/fetchngs/pull/385) - Add nf-core-utils plugin, replacing utils subworkflows +- [PR #385](https://github.com/nf-core/fetchngs/pull/385) - Update nf-schema and nf-test plugins +- [PR #385](https://github.com/nf-core/fetchngs/pull/385) - Simplify topic version handling ### Software dependencies @@ -57,6 +60,23 @@ Thank you to everyone else that has contributed by reporting bugs, enhancements | `sratools` | 2.11.0 | 3.2.1 | | `wget` | 1.20.1 | 1.25.0 | +### Plugins + +| Dependency | Old version | New version | +| ------------- | ----------- | ----------- | +| nf-core-utils | | 0.5.0 | +| nf-schema | 2.5.1 | 2.7.3 | +| nft-bam | 0.4.0 | 0.7.0 | +| nft-utils | 0.0.9 | 1.0.0 | + +### Modules / Subworkflows + +| Dependency | Old name | New name | +| ------------ | ----------------------- | -------- | +| Subworkflows | utils_nextflow_pipeline | | +| Subworkflows | utils_nfcore_pipeline | | +| Subworkflows | utils_nfschema_plugin | | + ## [[1.12.0](https://github.com/nf-core/fetchngs/releases/tag/1.12.0)] - 2024-02-29 ### :warning: Major enhancements diff --git a/modules.json b/modules.json index 80a85b2c..74c4669e 100644 --- a/modules.json +++ b/modules.json @@ -33,21 +33,6 @@ "branch": "master", "git_sha": "8e6139dbf4f3ca01bbac6152d2833e04edff18df", "installed_by": ["subworkflows"] - }, - "utils_nextflow_pipeline": { - "branch": "master", - "git_sha": "1a545fcbd762911c21a64ced3dbef99b2b51ac75", - "installed_by": ["subworkflows"] - }, - "utils_nfcore_pipeline": { - "branch": "master", - "git_sha": "a3fb7351b1fdb2b1de282b765816bbea190e86a8", - "installed_by": ["subworkflows"] - }, - "utils_nfschema_plugin": { - "branch": "master", - "git_sha": "a7b27fd25bfa8dcc07d299e88bd790585901a436", - "installed_by": ["subworkflows"] } } } diff --git a/nextflow.config b/nextflow.config index 824d8376..8a2e3790 100644 --- a/nextflow.config +++ b/nextflow.config @@ -327,7 +327,8 @@ manifest { // Nextflow plugins plugins { - id 'nf-schema@2.5.1' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-core-utils@0.5.0' // nf-core-utils is a collection of utilities for Nextflow pipelines + id 'nf-schema@2.7.3' // Validation of pipeline parameters and creation of an input channel from a sample sheet } validation { diff --git a/nf-test.config b/nf-test.config index 0dbaad56..ca6c95e3 100644 --- a/nf-test.config +++ b/nf-test.config @@ -33,8 +33,8 @@ config { // load the necessary plugins plugins { - load "nft-bam@0.4.0" + load "nft-bam@0.7.0" load "nft-csv@0.1.0" - load "nft-utils@0.0.9" + load "nft-utils@1.0.0" } } diff --git a/subworkflows/local/utils_nfcore_fetchngs_pipeline/main.nf b/subworkflows/local/utils_nfcore_fetchngs_pipeline/main.nf index 29b94e2d..666f02c4 100644 --- a/subworkflows/local/utils_nfcore_fetchngs_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_fetchngs_pipeline/main.nf @@ -8,14 +8,19 @@ ======================================================================================== */ -include { UTILS_NFSCHEMA_PLUGIN } from '../../nf-core/utils_nfschema_plugin' -include { paramsSummaryMap } from 'plugin/nf-schema' -include { samplesheetToList } from 'plugin/nf-schema' -include { paramsHelp } from 'plugin/nf-schema' -include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' -include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' -include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' -include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' +include { checkCondaChannels } from 'plugin/nf-core-utils' +include { checkConfigProvided } from 'plugin/nf-core-utils' +include { checkProfileProvided } from 'plugin/nf-core-utils' +include { completionEmail } from 'plugin/nf-core-utils' +include { completionSummary } from 'plugin/nf-core-utils' +include { dumpParametersToJSON } from 'plugin/nf-core-utils' +include { getWorkflowVersion } from 'plugin/nf-core-utils' + +include { paramsHelp } from 'plugin/nf-schema' +include { paramsSummaryLog } from 'plugin/nf-schema' +include { paramsSummaryMap } from 'plugin/nf-schema' +include { samplesheetToList } from 'plugin/nf-schema' +include { validateParameters } from 'plugin/nf-schema' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -41,12 +46,18 @@ workflow PIPELINE_INITIALISATION { // // Print version and exit if required and dump pipeline parameters to JSON file // - UTILS_NEXTFLOW_PIPELINE( - version, - true, - outdir, - workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1, - ) + if (version) { + log.info("${workflow.manifest.name} ${getWorkflowVersion()}") + System.exit(0) + } + + if (outdir) { + dumpParametersToJSON(outdir, params) + } + + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + checkCondaChannels() + } // // Validate parameters and generate parameter summary to stdout @@ -77,25 +88,37 @@ workflow PIPELINE_INITIALISATION { command = "nextflow run ${workflow.manifest.name} -profile --input samplesheet.csv --outdir " - UTILS_NFSCHEMA_PLUGIN( - workflow, - validate_params, - null, - help, - help_full, - show_hidden, - before_text, - after_text, - command, - null, - ) + if (help || help_full) { + help_options = [ + beforeText: before_text, + afterText: after_text, + command: command, + showHidden: show_hidden, + fullHelp: help_full, + ] + log.info( + paramsHelp( + help_options, + (help instanceof String && help != "true") ? help : "", + ) + ) + exit(0) + } + + checkProfileProvided(nextflow_cli_args) + + log.info(before_text) + log.info(paramsSummaryLog([:], workflow)) + log.info(after_text) + + if (validate_params) { + validateParameters([:]) + } // // Check config provided to the pipeline // - UTILS_NFCORE_PIPELINE( - nextflow_cli_args - ) + checkConfigProvided() // // Create channel from input file provided through params.input @@ -203,6 +226,7 @@ def sraCheckENAMetadataFields(ena_metadata_fields) { error("Invalid option: '${ena_metadata_fields}'. Minimally required fields for '--ena_metadata_fields': '${valid_ena_metadata_fields.join(',')}'") } } + // // Print a warning after pipeline has completed // diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf deleted file mode 100644 index 37939acd..00000000 --- a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf +++ /dev/null @@ -1,138 +0,0 @@ -// -// Subworkflow with functionality that may be useful for any Nextflow pipeline -// - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - SUBWORKFLOW DEFINITION -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -workflow UTILS_NEXTFLOW_PIPELINE { - take: - print_version // boolean: print version - dump_parameters // boolean: dump parameters - outdir // path: base directory used to publish pipeline results - check_conda_channels // boolean: check conda channels - - main: - - // - // Print workflow version and exit on --version - // - if (print_version) { - log.info("${workflow.manifest.name} ${getWorkflowVersion()}") - System.exit(0) - } - - // - // Dump pipeline parameters to a JSON file - // - if (dump_parameters && outdir) { - dumpParametersToJSON(outdir) - } - - // - // When running with Conda, warn if channels have not been set-up appropriately - // - if (check_conda_channels) { - checkCondaChannels() - } - - emit: - dummy_emit = true -} - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - FUNCTIONS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -// -// Generate version string -// -def getWorkflowVersion() { - def version_string = "" as String - if (workflow.manifest.version) { - def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' - version_string += "${prefix_v}${workflow.manifest.version}" - } - - if (workflow.commitId) { - def git_shortsha = workflow.commitId.substring(0, 7) - version_string += "-g${git_shortsha}" - } - - return version_string -} - -// -// Dump pipeline parameters to a JSON file -// -def dumpParametersToJSON(outdir) { - def timestamp = new java.util.Date().format('yyyy-MM-dd_HH-mm-ss') - def filename = "params_${timestamp}.json" - def temp_pf = workflow.launchDir.resolve(".${filename}") - def jsonGenerator = new groovy.json.JsonGenerator.Options() - .excludeNulls() - .addConverter(Path) { Path path -> path.toUriString() } - .addConverter(Duration) { Duration duration -> duration.toMillis() } - .addConverter(MemoryUnit) { MemoryUnit memory -> memory.toBytes() } - .addConverter(nextflow.script.types.VersionNumber) { nextflow.script.types.VersionNumber version -> version.toString() } - .build() - def jsonStr = jsonGenerator.toJson(params) - temp_pf.text = groovy.json.JsonOutput.prettyPrint(jsonStr) - if (outdir instanceof Path) { - temp_pf.copyTo(outdir.resolve("pipeline_info/${filename}")) - } else if (outdir instanceof String) { - temp_pf.copyTo("${outdir}/pipeline_info/params_${timestamp}.json") - } else { - log.warn("Could not determine type of outdir, parameters JSON file will not be copied to output directory!") - } - temp_pf.delete() -} - -// -// When running with -profile conda, warn if channels have not been set-up appropriately -// -def checkCondaChannels() { - def parser = new org.yaml.snakeyaml.Yaml() - def channels = [] - try { - def config = parser.load("conda config --show channels".execute().text) - channels = config.channels - } - catch (NullPointerException e) { - log.debug(e) - log.warn("Could not verify conda channel configuration.") - return null - } - catch (IOException e) { - log.debug(e) - log.warn("Could not verify conda channel configuration.") - return null - } - - // Check that all channels are present - // This channel list is ordered by required channel priority. - def required_channels_in_order = ['conda-forge', 'bioconda'] - def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean - - // Check that they are in the right order - def channel_priority_violation = required_channels_in_order != channels.findAll { ch -> ch in required_channels_in_order } - - if (channels_missing | channel_priority_violation) { - log.warn """\ - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - There is a problem with your Conda configuration! - You will need to set-up the conda-forge and bioconda channels correctly. - Please refer to https://bioconda.github.io/ - The observed channel order is - ${channels} - but the following channel order is required: - ${required_channels_in_order} - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - """.stripIndent(true) - } -} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml b/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml deleted file mode 100644 index e5c3a0a8..00000000 --- a/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml +++ /dev/null @@ -1,38 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json -name: "UTILS_NEXTFLOW_PIPELINE" -description: Subworkflow with functionality that may be useful for any Nextflow pipeline -keywords: - - utility - - pipeline - - initialise - - version -components: [] -input: - - print_version: - type: boolean - description: | - Print the version of the pipeline and exit - - dump_parameters: - type: boolean - description: | - Dump the parameters of the pipeline to a JSON file - - output_directory: - type: directory - description: Path to output dir to write JSON file to. - pattern: "results/" - - check_conda_channel: - type: boolean - description: | - Check if the conda channel priority is correct. -output: - - dummy_emit: - type: boolean - description: | - Dummy emit to make nf-core subworkflows lint happy -authors: - - "@adamrtalbot" - - "@drpatelh" -maintainers: - - "@adamrtalbot" - - "@drpatelh" - - "@maxulysse" diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf deleted file mode 100644 index afca5439..00000000 --- a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ /dev/null @@ -1,355 +0,0 @@ -// -// Subworkflow with utility functions specific to the nf-core pipeline template -// - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - SUBWORKFLOW DEFINITION -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -workflow UTILS_NFCORE_PIPELINE { - take: - nextflow_cli_args - - main: - valid_config = checkConfigProvided() - checkProfileProvided(nextflow_cli_args) - - emit: - valid_config = valid_config -} - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - FUNCTIONS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -// -// Warn if a -profile or Nextflow config has not been provided to run the pipeline -// -def checkConfigProvided() { - def valid_config = true as Boolean - if (workflow.profile == 'standard' && workflow.configFiles.size() <= 1) { - log.warn( - "[${workflow.manifest.name}] You are attempting to run the pipeline without any custom configuration!\n\n" + "This will be dependent on your local compute environment but can be achieved via one or more of the following:\n" + " (1) Using an existing pipeline profile e.g. `-profile docker` or `-profile singularity`\n" + " (2) Using an existing nf-core/configs for your Institution e.g. `-profile crick` or `-profile uppmax`\n" + " (3) Using your own local custom config e.g. `-c /path/to/your/custom.config`\n\n" + "Please refer to the quick start section and usage docs for the pipeline.\n " - ) - valid_config = false - } - return valid_config -} - -// -// Exit pipeline if --profile contains spaces -// -def checkProfileProvided(nextflow_cli_args) { - if (workflow.profile.endsWith(',')) { - error( - "The `-profile` option cannot end with a trailing comma, please remove it and re-run the pipeline!\n" + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" - ) - } - if (nextflow_cli_args[0]) { - log.warn( - "nf-core pipelines do not accept positional arguments. The positional argument `${nextflow_cli_args[0]}` has been detected.\n" + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" - ) - } -} - -// -// Generate workflow version string -// -def getWorkflowVersion() { - def version_string = "" as String - if (workflow.manifest.version) { - def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' - version_string += "${prefix_v}${workflow.manifest.version}" - } - - if (workflow.commitId) { - def git_shortsha = workflow.commitId.substring(0, 7) - version_string += "-g${git_shortsha}" - } - - return version_string -} - -// -// Get software versions for pipeline -// -def processVersionsFromYAML(yaml_file) { - def yaml = new org.yaml.snakeyaml.Yaml() - def versions = yaml.load(yaml_file).collectEntries { k, v -> [k.tokenize(':')[-1], v] } - return yaml.dumpAsMap(versions).trim() -} - -// -// Get workflow version for pipeline -// -def workflowVersionToYAML() { - return """ - Workflow: - ${workflow.manifest.name}: ${getWorkflowVersion()} - Nextflow: ${workflow.nextflow.version} - """.stripIndent().trim() -} - -// -// Get channel of software versions used in pipeline in YAML format -// -def softwareVersionsToYAML(ch_versions) { - return ch_versions.unique().map { version -> processVersionsFromYAML(version) }.unique().mix(channel.of(workflowVersionToYAML())) -} - -// -// Get workflow summary for MultiQC -// -def paramsSummaryMultiqc(summary_params) { - def summary_section = '' - summary_params - .keySet() - .each { group -> - def group_params = summary_params.get(group) - // This gets the parameters of that particular group - if (group_params) { - summary_section += "

${group}

\n" - summary_section += "
\n" - group_params - .keySet() - .sort() - .each { param -> - summary_section += "
${param}
${group_params.get(param) ?: 'N/A'}
\n" - } - summary_section += "
\n" - } - } - - def yaml_file_text = "id: '${workflow.manifest.name.replace('/', '-')}-summary'\n" as String - yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" - yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" - yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" - yaml_file_text += "plot_type: 'html'\n" - yaml_file_text += "data: |\n" - yaml_file_text += "${summary_section}" - - return yaml_file_text -} - -// -// ANSII colours used for terminal logging -// -def logColours(monochrome_logs=true) { - def colorcodes = [:] as Map - - // Reset / Meta - colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" - colorcodes['bold'] = monochrome_logs ? '' : "\033[1m" - colorcodes['dim'] = monochrome_logs ? '' : "\033[2m" - colorcodes['underlined'] = monochrome_logs ? '' : "\033[4m" - colorcodes['blink'] = monochrome_logs ? '' : "\033[5m" - colorcodes['reverse'] = monochrome_logs ? '' : "\033[7m" - colorcodes['hidden'] = monochrome_logs ? '' : "\033[8m" - - // Regular Colors - colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" - colorcodes['red'] = monochrome_logs ? '' : "\033[0;31m" - colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" - colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" - colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" - colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" - colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" - colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" - - // Bold - colorcodes['bblack'] = monochrome_logs ? '' : "\033[1;30m" - colorcodes['bred'] = monochrome_logs ? '' : "\033[1;31m" - colorcodes['bgreen'] = monochrome_logs ? '' : "\033[1;32m" - colorcodes['byellow'] = monochrome_logs ? '' : "\033[1;33m" - colorcodes['bblue'] = monochrome_logs ? '' : "\033[1;34m" - colorcodes['bpurple'] = monochrome_logs ? '' : "\033[1;35m" - colorcodes['bcyan'] = monochrome_logs ? '' : "\033[1;36m" - colorcodes['bwhite'] = monochrome_logs ? '' : "\033[1;37m" - - // Underline - colorcodes['ublack'] = monochrome_logs ? '' : "\033[4;30m" - colorcodes['ured'] = monochrome_logs ? '' : "\033[4;31m" - colorcodes['ugreen'] = monochrome_logs ? '' : "\033[4;32m" - colorcodes['uyellow'] = monochrome_logs ? '' : "\033[4;33m" - colorcodes['ublue'] = monochrome_logs ? '' : "\033[4;34m" - colorcodes['upurple'] = monochrome_logs ? '' : "\033[4;35m" - colorcodes['ucyan'] = monochrome_logs ? '' : "\033[4;36m" - colorcodes['uwhite'] = monochrome_logs ? '' : "\033[4;37m" - - // High Intensity - colorcodes['iblack'] = monochrome_logs ? '' : "\033[0;90m" - colorcodes['ired'] = monochrome_logs ? '' : "\033[0;91m" - colorcodes['igreen'] = monochrome_logs ? '' : "\033[0;92m" - colorcodes['iyellow'] = monochrome_logs ? '' : "\033[0;93m" - colorcodes['iblue'] = monochrome_logs ? '' : "\033[0;94m" - colorcodes['ipurple'] = monochrome_logs ? '' : "\033[0;95m" - colorcodes['icyan'] = monochrome_logs ? '' : "\033[0;96m" - colorcodes['iwhite'] = monochrome_logs ? '' : "\033[0;97m" - - // Bold High Intensity - colorcodes['biblack'] = monochrome_logs ? '' : "\033[1;90m" - colorcodes['bired'] = monochrome_logs ? '' : "\033[1;91m" - colorcodes['bigreen'] = monochrome_logs ? '' : "\033[1;92m" - colorcodes['biyellow'] = monochrome_logs ? '' : "\033[1;93m" - colorcodes['biblue'] = monochrome_logs ? '' : "\033[1;94m" - colorcodes['bipurple'] = monochrome_logs ? '' : "\033[1;95m" - colorcodes['bicyan'] = monochrome_logs ? '' : "\033[1;96m" - colorcodes['biwhite'] = monochrome_logs ? '' : "\033[1;97m" - - return colorcodes -} - -// Return a single report from an object that may be a Path or List -// -def getSingleReport(multiqc_reports) { - if (multiqc_reports instanceof Path) { - return multiqc_reports - } else if (multiqc_reports instanceof List) { - if (multiqc_reports.size() == 0) { - log.warn("[${workflow.manifest.name}] No reports found from process 'MULTIQC'") - return null - } else if (multiqc_reports.size() == 1) { - return multiqc_reports.first() - } else { - log.warn("[${workflow.manifest.name}] Found multiple reports from process 'MULTIQC', will use only one") - return multiqc_reports.first() - } - } else { - return null - } -} - -// -// Construct and send completion email -// -def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs=true, multiqc_report=null) { - - // Set up the e-mail variables - def subject = "[${workflow.manifest.name}] Successful: ${workflow.runName}" - if (!workflow.success) { - subject = "[${workflow.manifest.name}] FAILED: ${workflow.runName}" - } - - def summary = [:] - summary_params - .keySet() - .sort() - .each { group -> - summary << summary_params[group] - } - - def misc_fields = [:] - misc_fields['Date Started'] = workflow.start - misc_fields['Date Completed'] = workflow.complete - misc_fields['Pipeline script file path'] = workflow.scriptFile - misc_fields['Pipeline script hash ID'] = workflow.scriptId - if (workflow.repository) { - misc_fields['Pipeline repository Git URL'] = workflow.repository - } - if (workflow.commitId) { - misc_fields['Pipeline repository Git Commit'] = workflow.commitId - } - if (workflow.revision) { - misc_fields['Pipeline Git branch/tag'] = workflow.revision - } - misc_fields['Nextflow Version'] = workflow.nextflow.version - misc_fields['Nextflow Build'] = workflow.nextflow.build - misc_fields['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp - - def email_fields = [:] - email_fields['version'] = getWorkflowVersion() - email_fields['runName'] = workflow.runName - email_fields['success'] = workflow.success - email_fields['dateComplete'] = workflow.complete - email_fields['duration'] = workflow.duration - email_fields['exitStatus'] = workflow.exitStatus - email_fields['errorMessage'] = (workflow.errorMessage ?: 'None') - email_fields['errorReport'] = (workflow.errorReport ?: 'None') - email_fields['commandLine'] = workflow.commandLine - email_fields['projectDir'] = workflow.projectDir - email_fields['summary'] = summary << misc_fields - - // On success try attach the multiqc report - def mqc_report = getSingleReport(multiqc_report) - - // Check if we are only sending emails on failure - def email_address = email - if (!email && email_on_fail && !workflow.success) { - email_address = email_on_fail - } - - // Render the TXT template - def engine = new groovy.text.GStringTemplateEngine() - def tf = new File("${workflow.projectDir}/assets/email_template.txt") - def txt_template = engine.createTemplate(tf).make(email_fields) - def email_txt = txt_template.toString() - - // Render the HTML template - def hf = new File("${workflow.projectDir}/assets/email_template.html") - def html_template = engine.createTemplate(hf).make(email_fields) - def email_html = html_template.toString() - - // Render the sendmail template - def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as MemoryUnit - def smail_fields = [email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "${workflow.projectDir}", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes()] - def sf = new File("${workflow.projectDir}/assets/sendmail_template.txt") - def sendmail_template = engine.createTemplate(sf).make(smail_fields) - def sendmail_html = sendmail_template.toString() - - // Send the HTML e-mail - def colors = logColours(monochrome_logs) as Map - if (email_address) { - try { - if (plaintext_email) { - new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') - } - // Try to send HTML e-mail using sendmail - def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") - sendmail_tf.withWriter { w -> w << sendmail_html } - ['sendmail', '-t'].execute() << sendmail_html - log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Sent summary e-mail to ${email_address} (sendmail)-") - } - catch (Exception msg) { - log.debug(msg.toString()) - log.debug("Trying with mail instead of sendmail") - // Catch failures and try with plaintext - def mail_cmd = ['mail', '-s', subject, '--content-type=text/html', email_address] - mail_cmd.execute() << email_html - log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Sent summary e-mail to ${email_address} (mail)-") - } - } - - // Write summary e-mail HTML to a file - def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") - output_hf.withWriter { w -> w << email_html } - nextflow.extension.FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html") - output_hf.delete() - - // Write summary e-mail TXT to a file - def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") - output_tf.withWriter { w -> w << email_txt } - nextflow.extension.FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt") - output_tf.delete() -} - -// -// Print pipeline summary on completion -// -def completionSummary(monochrome_logs=true) { - def colors = logColours(monochrome_logs) as Map - if (workflow.success) { - if (workflow.stats.ignoredCount == 0) { - log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Pipeline completed successfully${colors.reset}-") - } - else { - log.info("-${colors.purple}[${workflow.manifest.name}]${colors.yellow} Pipeline completed successfully, but with errored process(es) ${colors.reset}-") - } - } - else { - log.info("-${colors.purple}[${workflow.manifest.name}]${colors.red} Pipeline completed with errors${colors.reset}-") - } -} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml b/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml deleted file mode 100644 index d08d2434..00000000 --- a/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml +++ /dev/null @@ -1,24 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json -name: "UTILS_NFCORE_PIPELINE" -description: Subworkflow with utility functions specific to the nf-core pipeline template -keywords: - - utility - - pipeline - - initialise - - version -components: [] -input: - - nextflow_cli_args: - type: list - description: | - Nextflow CLI positional arguments -output: - - success: - type: boolean - description: | - Dummy output to indicate success -authors: - - "@adamrtalbot" -maintainers: - - "@adamrtalbot" - - "@maxulysse" diff --git a/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/subworkflows/nf-core/utils_nfschema_plugin/main.nf deleted file mode 100644 index 9ff0681f..00000000 --- a/subworkflows/nf-core/utils_nfschema_plugin/main.nf +++ /dev/null @@ -1,77 +0,0 @@ -// -// Subworkflow that uses the nf-schema plugin to validate parameters and render the parameter summary -// - -include { paramsSummaryLog } from 'plugin/nf-schema' -include { validateParameters } from 'plugin/nf-schema' -include { paramsHelp } from 'plugin/nf-schema' - -workflow UTILS_NFSCHEMA_PLUGIN { - - take: - input_workflow // workflow: the workflow object used by nf-schema to get metadata from the workflow - validate_params // boolean: validate the parameters - parameters_schema // string: path to the parameters JSON schema. - // this has to be the same as the schema given to `validation.parametersSchema` - // when this input is empty it will automatically use the configured schema or - // "${projectDir}/nextflow_schema.json" as default. This input should not be empty - // for meta pipelines - help // boolean: show help message - help_full // boolean: show full help message - show_hidden // boolean: show hidden parameters in help message - before_text // string: text to show before the help message and parameters summary - after_text // string: text to show after the help message and parameters summary - command // string: an example command of the pipeline - cli_typecast // boolean: whether to perform typecasting of CLI parameters. Set this to `null` to use the default behaviour - - main: - - if(help || help_full) { - help_options = [ - beforeText: before_text, - afterText: after_text, - command: command, - showHidden: show_hidden, - fullHelp: help_full, - ] - if(parameters_schema) { - help_options << [parameters_schema: parameters_schema] - } - log.info paramsHelp( - help_options, - (help instanceof String && help != "true") ? help : "", - ) - exit 0 - } - - // - // Print parameter summary to stdout. This will display the parameters - // that differ from the default given in the JSON schema - // - - summary_options = [:] - if(parameters_schema) { - summary_options << [parameters_schema: parameters_schema] - } - log.info before_text - log.info paramsSummaryLog(summary_options, input_workflow) - log.info after_text - - // - // Validate the parameters using nextflow_schema.json or the schema - // given via the validation.parametersSchema configuration option - // - if(validate_params) { - validateOptions = [:] - if(parameters_schema) { - validateOptions << [parameters_schema: parameters_schema] - } - if(cli_typecast != null) { - validateOptions << [cast_cli_params: cli_typecast] - } - validateParameters(validateOptions) - } - - emit: - dummy_emit = true -} diff --git a/subworkflows/nf-core/utils_nfschema_plugin/meta.yml b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml deleted file mode 100644 index 1d8c75a9..00000000 --- a/subworkflows/nf-core/utils_nfschema_plugin/meta.yml +++ /dev/null @@ -1,59 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json -name: "utils_nfschema_plugin" -description: Run nf-schema to validate parameters and create a summary of changed parameters -keywords: - - validation - - JSON schema - - plugin - - parameters - - summary -components: [] -input: - - input_workflow: - type: object - description: | - The workflow object of the used pipeline. - This object contains meta data used to create the params summary log - - validate_params: - type: boolean - description: Validate the parameters and error if invalid. - - parameters_schema: - type: string - description: | - Path to the parameters JSON schema. - This has to be the same as the schema given to the `validation.parametersSchema` config - option. When this input is empty it will automatically use the configured schema or - "${projectDir}/nextflow_schema.json" as default. The schema should not be given in this way - for meta pipelines. - - help: - type: boolean, string - description: | - Show the help message and exit. When a parameter name is given, show the help message for that parameter instead of the general help message. - - help_full: - type: boolean - description: Show the full help message and exit. - - show_hidden: - type: boolean - description: Show hidden parameters in the help message. - - before_text: - type: string - description: Text to show before the parameters summary and help message. - - after_text: - type: string - description: Text to show after the parameters summary and help message. - - command: - type: string - description: An example command to run the pipeline, to show in the help message and the summary. - - cli_typecast: - type: boolean - description: | - Whether to apply typecasting to the parameters given via the CLI before validation. - Set this to `null` to use the default behavior. -output: - - dummy_emit: - type: boolean - description: Dummy emit to make nf-core subworkflows lint happy -authors: - - "@nvnieuwk" -maintainers: - - "@nvnieuwk" diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index 7b2a403e..3caebc3d 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -9,7 +9,7 @@ "python": "3.9.5" }, "SRATOOLS_FASTERQDUMP": { - "pigz": 2.8, + "pigz": "2.8", "sratools": "3.2.1" }, "SRATOOLS_PREFETCH": { diff --git a/workflows/fetchngs.nf b/workflows/fetchngs.nf index a9a33fa4..a5063df4 100644 --- a/workflows/fetchngs.nf +++ b/workflows/fetchngs.nf @@ -19,7 +19,7 @@ include { CHANNEL_SRA_CREATE_CSV } from '../subworkflows/l include { FASTQDL } from '../modules/nf-core/fastqdl' include { FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS } from '../subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools' -include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' +include { softwareVersionsToYAML } from 'plugin/nf-core-utils' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -131,31 +131,15 @@ workflow FETCHNGS { // // Collate and save software versions // - def topic_versions = channel.topic("versions") - .distinct() - .branch { entry -> - versions_file: entry instanceof Path - versions_tuple: true - } - - def topic_versions_string = topic_versions.versions_tuple - .map { process, tool, version -> - [process[process.lastIndexOf(':') + 1..-1], " ${tool}: ${version}"] - } - .groupTuple(by: 0) - .map { process, tool_versions -> - tool_versions.unique().sort() - "${process}:\n${tool_versions.join('\n')}" - } - - def ch_collated_versions = softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) - .mix(topic_versions_string) - .collectFile( - storeDir: "${outdir}/pipeline_info", - name: 'nf_core_' + 'fetchngs_software_' + 'versions.yml', - sort: true, - newLine: true, - ) + def ch_collated_versions = softwareVersionsToYAML( + softwareVersions: channel.topic("versions"), + nextflowVersion: workflow.nextflow.version, + ).collectFile( + storeDir: "${outdir}/pipeline_info", + name: 'nf_core_' + 'fetchngs_software_' + 'versions.yml', + sort: true, + newLine: true, + ) emit: samplesheet = ch_samplesheet diff --git a/workflows/tests/sra_download_method_sratools.nf.test.snap b/workflows/tests/sra_download_method_sratools.nf.test.snap index d2ca0d75..5afe921b 100644 --- a/workflows/tests/sra_download_method_sratools.nf.test.snap +++ b/workflows/tests/sra_download_method_sratools.nf.test.snap @@ -141,7 +141,7 @@ "python": "3.9.5" }, "SRATOOLS_FASTERQDUMP": { - "pigz": 2.8, + "pigz": "2.8", "sratools": "3.2.1" }, "SRATOOLS_PREFETCH": {