diff --git a/.github/snapshots/apple_container.nf.test.snap b/.github/snapshots/apple_container.nf.test.snap new file mode 100644 index 0000000000..5435135a0f --- /dev/null +++ b/.github/snapshots/apple_container.nf.test.snap @@ -0,0 +1,114 @@ +{ + "-profile test": { + "content": [ + { + "FASTQC": { + "fastqc": "0.12.1" + }, + "Workflow": { + "my-prefix/testpipeline": "v1.0.0dev" + } + }, + [ + "fastqc", + "fastqc/SAMPLE1_PE_1_fastqc.html", + "fastqc/SAMPLE1_PE_1_fastqc.zip", + "fastqc/SAMPLE1_PE_2_fastqc.html", + "fastqc/SAMPLE1_PE_2_fastqc.zip", + "fastqc/SAMPLE2_PE_1_fastqc.html", + "fastqc/SAMPLE2_PE_1_fastqc.zip", + "fastqc/SAMPLE2_PE_2_fastqc.html", + "fastqc/SAMPLE2_PE_2_fastqc.zip", + "fastqc/SAMPLE3_SE_1_fastqc.html", + "fastqc/SAMPLE3_SE_1_fastqc.zip", + "fastqc/SAMPLE3_SE_2_fastqc.html", + "fastqc/SAMPLE3_SE_2_fastqc.zip", + "multiqc", + "multiqc/multiqc_data", + "multiqc/multiqc_data/fastqc-status-check-heatmap.txt", + "multiqc/multiqc_data/fastqc_overrepresented_sequences_plot.txt", + "multiqc/multiqc_data/fastqc_per_base_n_content_plot.txt", + "multiqc/multiqc_data/fastqc_per_base_sequence_quality_plot.txt", + "multiqc/multiqc_data/fastqc_per_sequence_gc_content_plot_Counts.txt", + "multiqc/multiqc_data/fastqc_per_sequence_gc_content_plot_Percentages.txt", + "multiqc/multiqc_data/fastqc_per_sequence_quality_scores_plot.txt", + "multiqc/multiqc_data/fastqc_sequence_counts_plot.txt", + "multiqc/multiqc_data/fastqc_sequence_duplication_levels_plot.txt", + "multiqc/multiqc_data/fastqc_sequence_length_distribution_plot.txt", + "multiqc/multiqc_data/fastqc_top_overrepresented_sequences_table.txt", + "multiqc/multiqc_data/llms-full.txt", + "multiqc/multiqc_data/multiqc.log", + "multiqc/multiqc_data/multiqc.parquet", + "multiqc/multiqc_data/multiqc_citations.txt", + "multiqc/multiqc_data/multiqc_data.json", + "multiqc/multiqc_data/multiqc_fastqc.txt", + "multiqc/multiqc_data/multiqc_general_stats.txt", + "multiqc/multiqc_data/multiqc_software_versions.txt", + "multiqc/multiqc_data/multiqc_sources.txt", + "multiqc/multiqc_plots", + "multiqc/multiqc_plots/pdf", + "multiqc/multiqc_plots/pdf/fastqc-status-check-heatmap.pdf", + "multiqc/multiqc_plots/pdf/fastqc_overrepresented_sequences_plot.pdf", + "multiqc/multiqc_plots/pdf/fastqc_per_base_n_content_plot.pdf", + "multiqc/multiqc_plots/pdf/fastqc_per_base_sequence_quality_plot.pdf", + "multiqc/multiqc_plots/pdf/fastqc_per_sequence_gc_content_plot_Counts.pdf", + "multiqc/multiqc_plots/pdf/fastqc_per_sequence_gc_content_plot_Percentages.pdf", + "multiqc/multiqc_plots/pdf/fastqc_per_sequence_quality_scores_plot.pdf", + "multiqc/multiqc_plots/pdf/fastqc_sequence_counts_plot-cnt.pdf", + "multiqc/multiqc_plots/pdf/fastqc_sequence_counts_plot-pct.pdf", + "multiqc/multiqc_plots/pdf/fastqc_sequence_duplication_levels_plot.pdf", + "multiqc/multiqc_plots/pdf/fastqc_sequence_length_distribution_plot.pdf", + "multiqc/multiqc_plots/pdf/fastqc_top_overrepresented_sequences_table.pdf", + "multiqc/multiqc_plots/png", + "multiqc/multiqc_plots/png/fastqc-status-check-heatmap.png", + "multiqc/multiqc_plots/png/fastqc_overrepresented_sequences_plot.png", + "multiqc/multiqc_plots/png/fastqc_per_base_n_content_plot.png", + "multiqc/multiqc_plots/png/fastqc_per_base_sequence_quality_plot.png", + "multiqc/multiqc_plots/png/fastqc_per_sequence_gc_content_plot_Counts.png", + "multiqc/multiqc_plots/png/fastqc_per_sequence_gc_content_plot_Percentages.png", + "multiqc/multiqc_plots/png/fastqc_per_sequence_quality_scores_plot.png", + "multiqc/multiqc_plots/png/fastqc_sequence_counts_plot-cnt.png", + "multiqc/multiqc_plots/png/fastqc_sequence_counts_plot-pct.png", + "multiqc/multiqc_plots/png/fastqc_sequence_duplication_levels_plot.png", + "multiqc/multiqc_plots/png/fastqc_sequence_length_distribution_plot.png", + "multiqc/multiqc_plots/png/fastqc_top_overrepresented_sequences_table.png", + "multiqc/multiqc_plots/svg", + "multiqc/multiqc_plots/svg/fastqc-status-check-heatmap.svg", + "multiqc/multiqc_plots/svg/fastqc_overrepresented_sequences_plot.svg", + "multiqc/multiqc_plots/svg/fastqc_per_base_n_content_plot.svg", + "multiqc/multiqc_plots/svg/fastqc_per_base_sequence_quality_plot.svg", + "multiqc/multiqc_plots/svg/fastqc_per_sequence_gc_content_plot_Counts.svg", + "multiqc/multiqc_plots/svg/fastqc_per_sequence_gc_content_plot_Percentages.svg", + "multiqc/multiqc_plots/svg/fastqc_per_sequence_quality_scores_plot.svg", + "multiqc/multiqc_plots/svg/fastqc_sequence_counts_plot-cnt.svg", + "multiqc/multiqc_plots/svg/fastqc_sequence_counts_plot-pct.svg", + "multiqc/multiqc_plots/svg/fastqc_sequence_duplication_levels_plot.svg", + "multiqc/multiqc_plots/svg/fastqc_sequence_length_distribution_plot.svg", + "multiqc/multiqc_plots/svg/fastqc_top_overrepresented_sequences_table.svg", + "multiqc/multiqc_report.html", + "pipeline_info", + "pipeline_info/testpipeline_software_mqc_versions.yml" + ], + [ + "fastqc-status-check-heatmap.txt:md5,0f1975c565a16bf09be08a05c204ded7", + "fastqc_overrepresented_sequences_plot.txt:md5,4b23cea39c4e23deef6b97810bc1ee46", + "fastqc_per_base_n_content_plot.txt:md5,037692101c0130c72493d3bbfa3afac1", + "fastqc_per_base_sequence_quality_plot.txt:md5,bfe735f3e31befe13bdf6761bb297d6e", + "fastqc_per_sequence_gc_content_plot_Counts.txt:md5,7108d19c46ef7883e864ba274c457d2e", + "fastqc_per_sequence_gc_content_plot_Percentages.txt:md5,23f527c80a148e4f34e5a43f6e520a90", + "fastqc_per_sequence_quality_scores_plot.txt:md5,a0cc0e6df7bfb05257da1cfc88b13c50", + "fastqc_sequence_counts_plot.txt:md5,c6e4e1588e6765fe8df27812a1322fbd", + "fastqc_sequence_duplication_levels_plot.txt:md5,3cde2db4033f6c64648976d1174db925", + "fastqc_sequence_length_distribution_plot.txt:md5,e82b9b14a7e24c0c5f27af97cebb6870", + "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", + "multiqc_fastqc.txt:md5,1a41c2158adc9947bff9232962f70110", + "multiqc_general_stats.txt:md5,0b54e4e764665bd57fe0f95216744a78" + ] + ], + "meta": { + "nf-test": "0.9.3", + "nextflow": "25.10.4" + }, + "timestamp": "2025-06-16T14:29:10.076573" + } +} diff --git a/nf_core/__main__.py b/nf_core/__main__.py index bdf60b8cda..3e9f88a098 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -342,7 +342,7 @@ def command_pipelines_lint( @click.option( "-s", "--container-system", - type=click.Choice(["none", "singularity", "docker", "apptainer"]), + type=click.Choice(["none", "singularity", "docker", "apptainer", "apple_container"]), help="Download container images of required software.", ) @click.option( @@ -1303,7 +1303,7 @@ def command_modules_create( ) @click.option( "--profile", - type=click.Choice(["docker", "singularity", "conda"]), + type=click.Choice(["docker", "singularity", "conda", "apple_container", "apple_container_wave"]), default=None, help="Run tests with a specific profile", ) @@ -1665,7 +1665,7 @@ def command_subworkflows_create(ctx, subworkflow, directory, author, force): ) @click.option( "--profile", - type=click.Choice(["docker", "singularity", "conda"]), + type=click.Choice(["docker", "singularity", "conda", "apple_container", "apple_container_wave"]), default=None, help="Run tests with a specific profile", ) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index 587c724384..a0b9eddab4 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -145,10 +145,16 @@ def check_inputs(self) -> None: "type": "list", "name": "profile", "message": "Choose container software to run the test with", - "choices": ["Docker", "Singularity", "Conda"], + "choices": ["Docker", "Singularity", "Conda", "Apple Container", "Apple Container (Wave/arm64)"], } answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) - profile = answer["profile"].lower() + profile = answer["profile"] + # Map display names to Nextflow profile names + profile_map = { + "Apple Container": "apple_container", + "Apple Container (Wave/arm64)": "apple_container_wave", + } + profile = profile_map.get(profile, profile.lower()) os.environ["PROFILE"] = profile def display_nftest_output(self, nftest_out: bytes, nftest_err: bytes) -> None: diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 57908aa578..952e277760 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -26,6 +26,7 @@ [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) +[![run with apple container](https://img.shields.io/badge/run%20with-apple%20container-000000.svg?labelColor=000000&logo=apple)](https://github.com/apple/container) [![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://cloud.seqera.io/launch?pipeline=https://github.com/{{ name }}) {% endif -%} diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index ca7e7748fd..c34f4f56d2 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -164,10 +164,16 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to be used with [Charliecloud](https://charliecloud.io/) - `apptainer` - A generic configuration profile to be used with [Apptainer](https://apptainer.org/) +- `apple_container` + - A generic configuration profile to be used with [Apple container](https://github.com/apple/container) on macOS (Apple silicon) + - Emulates amd64 images by default so standard nf-core modules run without any Wave dependency. Set `--rosetta` to use faster Rosetta emulation (requires Rosetta installed on the host. +- `apple_container_wave` + - An Apple container profile that provisions native arm64 images from conda via [Wave](https://seqera.io/wave/) instead of emulating amd64. Faster per task when an arm64 build exists, but only works for tools that have an arm64 build available (conda `linux-aarch64` or a multi-arch image). + - Can be used on its own (`-profile apple_container_wave`) or composed after `apple_container` (`-profile apple_container,apple_container_wave`). It freezes and pushes built images to a registry, so set `TOWER_ACCESS_TOKEN` for durable, reproducible builds. - `wave` - A generic configuration profile to enable [Wave](https://seqera.io/wave/) containers. Use together with one of the above (requires Nextflow `24.03.0-edge` or later). - `conda` - - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter, Charliecloud, or Apptainer. + - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter, Charliecloud, Apptainer, or Apple container. ### `-resume` diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 34806e4729..4111875c09 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -51,6 +51,14 @@ params { {%- if test_config %} pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/'{% endif %} trace_report_suffix = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') + {%- if apple_container %} + + // Apple Container options + // When true, the apple_container profile emulates amd64 images with Rosetta + // (faster) instead of the default plain amd64 emulation. Requires Rosetta + // installed in the host OS. + apple_rosetta = false + {%- endif %} {%- if nf_core_configs %} // Config options @@ -107,8 +115,9 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false - conda.channels = ['conda-forge', 'bioconda'] apptainer.enabled = false + appleContainer.enabled = false + conda.channels = ['conda-forge', 'bioconda'] } mamba { conda.enabled = true @@ -119,6 +128,7 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false + appleContainer.enabled = false } docker { docker.enabled = true @@ -128,6 +138,7 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false + appleContainer.enabled = false docker.runOptions = '-u $(id -u):$(id -g)' } arm64 { @@ -153,6 +164,7 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false + appleContainer.enabled = false } podman { podman.enabled = true @@ -162,6 +174,7 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false + appleContainer.enabled = false } shifter { shifter.enabled = true @@ -171,6 +184,7 @@ profiles { podman.enabled = false charliecloud.enabled = false apptainer.enabled = false + appleContainer.enabled = false } charliecloud { charliecloud.enabled = true @@ -180,6 +194,7 @@ profiles { podman.enabled = false shifter.enabled = false apptainer.enabled = false + appleContainer.enabled = false } apptainer { apptainer.enabled = true @@ -190,6 +205,50 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false + appleContainer.enabled = false + } + apple_container { + appleContainer.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false + // Apple Container runs natively on Apple Silicon (arm64), but like + // Docker it can also run amd64 images via emulation. Since most nf-core + // modules ship amd64 container images, emulate amd64 by default so + // standard pipelines run without any Wave dependency. Set + // `--apple_rosetta` to use faster Rosetta emulation (requires Rosetta + // installed in the Apple Container VM). + process.arch = 'linux/amd64' + appleContainer.runOptions = params.apple_rosetta ? '--platform linux/amd64 --rosetta' : '--platform linux/amd64' + } + apple_container_wave { + // Self-contained Apple Container + Wave profile: provision native arm64 + // images from conda via Wave instead of emulating amd64. Usable on its + // own (`-profile apple_container_wave`, e.g. with + // `nf-core modules test --profile apple_container`-style single-profile + // callers) or composed after apple_container + // (`-profile apple_container,apple_container_wave`). Faster per task when + // an arm64 build exists, but only works for tools with an arm64 (conda + // linux-aarch64 / multi-arch) build available, and pushes frozen images + // to a registry (set TOWER_ACCESS_TOKEN for durable, reproducible + // builds). + appleContainer.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false + process.arch = 'arm64' + appleContainer.runOptions = '' + wave.enabled = true + wave.freeze = true + wave.strategy = 'conda,container' } wave { apptainer.ociAutoPull = true @@ -236,14 +295,15 @@ includeConfig params.custom_config_base && (!System.getenv('NXF_OFFLINE') || !pa // includeConfig params.custom_config_base && (!System.getenv('NXF_OFFLINE') || !params.custom_config_base.startsWith('http')) ? "${params.custom_config_base}/pipeline/{{ short_name }}.config" : "/dev/null" {%- endif %} -// Set default registry for Apptainer, Docker, Podman, Charliecloud and Singularity independent of -profile -// Will not be used unless Apptainer / Docker / Podman / Charliecloud / Singularity are enabled +// Set default registry for Apptainer, Docker, Podman, Charliecloud, Singularity and Apple container independent of -profile +// Will not be used unless Apptainer / Docker / Podman / Charliecloud / Singularity / Apple container are enabled // Set to your registry if you have a mirror of containers -apptainer.registry = 'quay.io' -docker.registry = 'quay.io' -podman.registry = 'quay.io' -singularity.registry = 'quay.io' -charliecloud.registry = 'quay.io' +apptainer.registry = 'quay.io' +appleContainer.registry = 'quay.io' +docker.registry = 'quay.io' +podman.registry = 'quay.io' +singularity.registry = 'quay.io' +charliecloud.registry = 'quay.io' {% if igenomes -%} // Load igenomes.config if required diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index a6caee0b3e..a1eb7f18af 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -226,7 +226,14 @@ "fa_icon": "far calendar", "description": "Suffix to add to the trace report filename. Default is the date and time in the format yyyy-MM-dd_HH-mm-ss.", "hidden": true - }{% if nf_schema %}, + }{% if apple_container %}, + "apple_rosetta": { + "type": "boolean", + "fa_icon": "fab fa-apple", + "description": "Use faster Rosetta amd64 emulation in the apple_container profile instead of plain amd64 emulation.", + "help_text": "Only applies to the `apple_container` profile. When `true`, amd64 container images are emulated with Rosetta (faster) instead of the default plain amd64 emulation. Requires Rosetta to be installed on the host.", + "hidden": true + }{% endif %}{% if nf_schema %}, "help": { "type": ["boolean", "string"], "description": "Display the help message." diff --git a/nf_core/pipelines/containers_utils.py b/nf_core/pipelines/containers_utils.py index d7de4fe73c..d0f88f8b39 100644 --- a/nf_core/pipelines/containers_utils.py +++ b/nf_core/pipelines/containers_utils.py @@ -17,6 +17,8 @@ "singularity_oras_arm64": ["singularity", "linux/arm64", "name"], "singularity_https_amd64": ["singularity", "linux/amd64", "https"], "singularity_https_arm64": ["singularity", "linux/arm64", "https"], + "appleContainer_amd64": ["docker", "linux/amd64", "name"], + "appleContainer_arm64": ["docker", "linux/arm64", "name"], "conda_lock_files_amd64": ["conda", "linux/amd64", "lock_file"], "conda_lock_files_arm64": ["conda", "linux/arm64", "lock_file"], } diff --git a/nf_core/pipelines/create/template_features.yml b/nf_core/pipelines/create/template_features.yml index eedd517933..d51d81e1e3 100644 --- a/nf_core/pipelines/create/template_features.yml +++ b/nf_core/pipelines/create/template_features.yml @@ -391,6 +391,49 @@ configurations: custom_pipelines: True default: False + apple_container: + skippable_paths: False + short_description: "Use Apple Container" + description: "Add Apple Container support to the pipeline" + help_text: | + This will add an `apple_container` profile to the pipeline configuration, enabling + native container execution on macOS with Apple Silicon via the + [Apple Container](https://github.com/apple/container) runtime. + + Apple Container uses standard OCI/Docker images and provides its own CLI + (`container`) with native `pull` and `save` commands. No Docker Desktop + installation is required. + + Apple Container runs natively on Apple Silicon (arm64), but like Docker it + can also run amd64 images via emulation. Because most nf-core modules ship + amd64 container images, the `apple_container` profile emulates amd64 by + default (`process.arch = 'linux/amd64'`, `--platform linux/amd64`), so + standard pipelines run out of the box with no Wave dependency and no + Seqera token. + + For faster emulation, set `--apple_rosetta` to run images through Rosetta + (`--rosetta`). This requires Rosetta to be installed on the host. + + If you would rather run native ARM64 images built from conda packages via + Wave, compose the opt-in `apple_container_wave` profile alongside it: + + ```bash + nextflow run -profile apple_container,apple_container_wave + ``` + + This is faster per task when a native arm64 build exists, but only works + for tools that have an ARM64 build available (a conda `linux-aarch64` + package or a multi-arch container image); tools whose upstream only ships + amd64 builds cannot be provisioned this way. It also sets `wave.freeze = + true`, so frozen images are pushed to a registry — set `TOWER_ACCESS_TOKEN` + for durable, reproducible builds. + + The `nf-core pipelines download` command will also support downloading + images for offline use with `--container-system appleContainer`. + nfcore_pipelines: True + custom_pipelines: True + default: true + development_environments: name: "Development Environments" features: diff --git a/nf_core/pipelines/download/apple_container.py b/nf_core/pipelines/download/apple_container.py new file mode 100644 index 0000000000..26500f09ea --- /dev/null +++ b/nf_core/pipelines/download/apple_container.py @@ -0,0 +1,153 @@ +"""Apple Container fetcher for nf-core download. + +Apple Container (https://github.com/apple/container) is a lightweight +container runtime for macOS on Apple Silicon. It uses standard OCI/Docker +images and provides its own CLI (``container``) with native ``pull`` and +``save`` commands for offline bundling. + +Images are pulled and saved using the ``container`` CLI directly — no +Docker installation is required. Loading images on the target machine +uses ``container image load``. +""" + +import logging +import shutil +from collections.abc import Iterable +from pathlib import Path + +import rich.console + +import nf_core.utils +from nf_core.pipelines.download.container_fetcher import ContainerFetcher +from nf_core.pipelines.download.docker import DockerFetcher +from nf_core.pipelines.download.utils import copy_container_load_scripts +from nf_core.utils import ContainerRegistryUrls + +log = logging.getLogger(__name__) +stderr = rich.console.Console( + stderr=True, + highlight=False, + force_terminal=nf_core.utils.rich_force_colors(), +) + + +class AppleContainerFetcher(DockerFetcher): + """ + Fetcher for Apple Container images. + + Subclasses :class:`DockerFetcher` — uses the native Apple Container CLI + (``container image pull`` / ``container image save``) instead of Docker. + The cleanup step writes an Apple-Container-specific load script. + """ + + def __init__( + self, + outdir: Path, + container_library: Iterable[str], + registry_set: Iterable[str], + parallel: int = 4, + hide_progress: bool = False, + image_arch: str = "linux/amd64", + ): + super().__init__( + outdir=outdir, + container_library=container_library, + registry_set=registry_set, + parallel=parallel, + hide_progress=hide_progress, + ) + # Override the container output directory name + self._container_output_dir = outdir / "apple-container-images" + # Architecture to request when pulling/saving images. Apple Container runs + # amd64 images via emulation (matching the apple_container profile default), + # so default to linux/amd64. Set to linux/arm64 for native arm64 images + # (parity with the opt-in apple_container_wave profile). + self.image_arch = image_arch + + def check_and_set_implementation(self) -> None: + """ + Check if Apple Container CLI is installed and set the implementation. + """ + container_binary = shutil.which("container") + if not container_binary: + raise OSError( + "Apple Container CLI ('container') is needed to pull images, " + "but it is not installed or not in $PATH.\n" + "See: https://github.com/apple/container" + ) + + self.implementation = "container" + + def construct_pull_command(self, address: str) -> list[str]: + """ + Construct the command to pull an image using Apple Container CLI. + + Args: + address (str): The address of the container to pull. + """ + pull_command = ["container", "image", "pull", "--platform", self.image_arch, address] + log.debug(f"Apple Container command: {' '.join(pull_command)}") + return pull_command + + def construct_save_command(self, output_path: Path, address: str) -> list[str]: + """ + Construct the command to save an image using Apple Container CLI. + + Args: + output_path (Path): The path to save the container image. + address (str): The address of the container to save. + """ + save_command = [ + "container", + "image", + "save", + "--platform", + self.image_arch, + "--output", + str(output_path), + address, + ] + log.debug(f"Apple Container command: {' '.join(save_command)}") + return save_command + + def gather_registries(self, workflow_directory: Path) -> set[str]: + """ + Gather the registries for Apple Container downloads. + + Checks ``appleContainer.registry``, ``docker.registry``, and + ``podman.registry`` keys from the workflow configuration. + """ + registry_set = self.base_registry_set.copy() + configured_registry_keys = ["appleContainer.registry", "docker.registry", "podman.registry"] + + registry_set |= self.gather_config_registries( + workflow_directory, + configured_registry_keys, + ) + + # Add the Seqera Docker container registry + registry_set.add(ContainerRegistryUrls.SEQERA_DOCKER.value) + return registry_set + + def cleanup(self) -> None: + """ + Write the Apple Container load message (skipping the Docker-specific one). + """ + # Call the grandparent cleanup directly to skip DockerFetcher.cleanup() + ContainerFetcher.cleanup(self) + self._write_apple_container_load_message() + + def _write_apple_container_load_message(self) -> None: + """ + Inform the user how to load downloaded images into Apple Container. + """ + img_dir = self.get_container_output_dir() + apple_load_script, _ = copy_container_load_scripts("appleContainer", img_dir) + indent = " " + stderr.print( + "\n" + f"{indent}Downloaded container images written to [magenta]'{img_dir}'[/].\n" + f"{indent}After copying the pipeline and images to the target macOS machine, run\n\n" + f"{indent}{indent}[green]./{apple_load_script}[/]\n\n" + f"{indent}inside [magenta]'{img_dir}'[/] to load the images into Apple Container.\n" + ) diff --git a/nf_core/pipelines/download/docker.py b/nf_core/pipelines/download/docker.py index 247a1579ba..6c3bce6e40 100644 --- a/nf_core/pipelines/download/docker.py +++ b/nf_core/pipelines/download/docker.py @@ -100,7 +100,7 @@ def gather_registries(self, workflow_directory: Path) -> set[str]: set[str]: The set of registries to use for the container download """ registry_set = self.base_registry_set.copy() - configured_registry_keys = ["docker.registry", "podman.registry"] + configured_registry_keys = ["appleContainer.registry", "docker.registry", "podman.registry"] # Add the registries defined in the workflow config registry_set |= self.gather_config_registries( diff --git a/nf_core/pipelines/download/download.py b/nf_core/pipelines/download/download.py index ef3b84dfcf..fbcd3386a7 100644 --- a/nf_core/pipelines/download/download.py +++ b/nf_core/pipelines/download/download.py @@ -21,6 +21,7 @@ import nf_core.pipelines.list import nf_core.utils from nf_core.github_api import gh_api +from nf_core.pipelines.download.apple_container import AppleContainerFetcher from nf_core.pipelines.download.container_fetcher import ContainerFetcher from nf_core.pipelines.download.docker import DockerFetcher from nf_core.pipelines.download.singularity import SINGULARITY_CACHE_DIR_ENV_VAR, SingularityFetcher @@ -87,12 +88,12 @@ def __init__( ): # Verify that the flags provided make sense together if ( - container_system == "docker" + container_system in ("docker", "apple_container") and container_cache_utilisation != "copy" and container_cache_utilisation is not None ): raise DownloadError( - "Only the 'copy' option for --container-cache-utilisation is supported for Docker images. " + "Only the 'copy' option for --container-cache-utilisation is supported for Docker/Apple container images. " ) self._pipeline = pipeline @@ -333,7 +334,7 @@ def download_workflow_static(self) -> None: raise DownloadError("Error editing pipeline config file to use local configs!") from e # Collect all required container images - if self.container_system in {"singularity", "docker", "apptainer"}: + if self.container_system in ("singularity", "apptainer", "docker", "apple_container"): workflow_directory = self.outdir / revision_dirname self.find_container_images(workflow_directory, revision) @@ -369,11 +370,11 @@ def download_workflow_platform(self, location: Path | None = None) -> None: self.workflow_repo.bare_clone(self.output_filename) # extract the required containers - if self.container_system in {"singularity", "docker", "apptainer"}: + if self.container_system in ("singularity", "apptainer", "docker", "apple_container"): for revision, commit in self.wf_sha.items(): # Checkout the repo in the current revision self.workflow_repo.checkout(commit) - # Collect all required singularity images + # Collect all required container images workflow_directory = self.workflow_repo.access() self.find_container_images(workflow_directory, revision) @@ -505,7 +506,7 @@ def prompt_container_download(self) -> None: stderr.print("\nIn addition to the pipeline code, this tool can download software containers.") self.container_system = questionary.select( "Download software container images:", - choices=["none", "singularity", "docker", "apptainer"], + choices=["none", "singularity", "docker", "apptainer", "apple_container"], style=nf_core.utils.nfcore_question_style, ).unsafe_ask() @@ -526,6 +527,14 @@ def setup_container_fetcher(self) -> None: hide_progress=self.hide_progress, container_system=self.container_system, ) + elif self.container_system == "apple_container": + self.container_fetcher = AppleContainerFetcher( + outdir=self.outdir, + registry_set=self.registry_set, + container_library=self.container_library, + parallel=self.parallel, + hide_progress=self.hide_progress, + ) elif self.container_system == "docker": self.container_fetcher = DockerFetcher( outdir=self.outdir, diff --git a/nf_core/pipelines/download/load_scripts/appleContainer-load.sh b/nf_core/pipelines/download/load_scripts/appleContainer-load.sh new file mode 100644 index 0000000000..5621349210 --- /dev/null +++ b/nf_core/pipelines/download/load_scripts/appleContainer-load.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail # Ensure that the script exits as early as possible + +LOGFILE="appleContainer-load.log" + +# Clear log +> "$LOGFILE" + +if ! command -v container &> /dev/null +then + echo "Error: Apple Container CLI ('container') is not installed. Please install it to continue." >&2 + echo "See: https://github.com/apple/container" >&2 + exit 1 +fi + +echo "Loading tar archives into Apple Container" +for tarfile in $(ls -1 *.tar); do + if output=$(container load < "$tarfile" 2>&1); then + echo "SUCCESS: $tarfile" + echo "SUCCESS: $tarfile" >> "$LOGFILE" + echo "$output" >> "$LOGFILE" + echo "----------------------------------------------------------------" >> "$LOGFILE" + else + echo "ERROR: $tarfile" + echo "ERROR: $tarfile" >> "$LOGFILE" + echo "$output" >> "$LOGFILE" + echo "----------------------------------------------------------------" >> "$LOGFILE" + fi +done diff --git a/nf_core/pipelines/download/singularity.py b/nf_core/pipelines/download/singularity.py index 98d4b5981a..3f57912953 100644 --- a/nf_core/pipelines/download/singularity.py +++ b/nf_core/pipelines/download/singularity.py @@ -228,6 +228,7 @@ def gather_registries(self, workflow_directory: Path) -> set[str]: # Select registries defined in pipeline config configured_registry_keys = [ + "appleContainer.registry", "apptainer.registry", "docker.registry", "podman.registry", diff --git a/tests/data/mock_pipeline_containers/nextflow.config b/tests/data/mock_pipeline_containers/nextflow.config index d07c04a405..9c881b2e07 100644 --- a/tests/data/mock_pipeline_containers/nextflow.config +++ b/tests/data/mock_pipeline_containers/nextflow.config @@ -149,6 +149,24 @@ profiles { shifter.enabled = false charliecloud.enabled = false } + apple_container { + appleContainer.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false + // The production template's apple_container profile emulates amd64 images + // (process.arch = 'linux/amd64', --platform linux/amd64) and does NOT + // enable Wave by default. The runOptions/process.arch settings are + // omitted here because this fixture exercises container discovery via + // `nextflow inspect`: like the docker/apptainer/singularity profiles + // above, it only enables the container system so inspection can resolve + // the declared images. The opt-in apple_container_wave profile (Wave/arm64) + // is likewise omitted so inspection never depends on the live Wave API. + } wave { apptainer.ociAutoPull = true singularity.ociAutoPull = true @@ -167,9 +185,10 @@ profiles { -// Set default registry for Apptainer, Docker, Podman, Charliecloud and Singularity independent of -profile -// Will not be used unless Apptainer / Docker / Podman / Charliecloud / Singularity are enabled +// Set default registry for Apptainer, Docker, Podman, Charliecloud, Singularity and Apple Container independent of -profile +// Will not be used unless Apptainer / Docker / Podman / Charliecloud / Singularity / Apple Container are enabled // Set to your registry if you have a mirror of containers +appleContainer.registry = 'quay.io' apptainer.registry = 'quay.io' docker.registry = 'quay.io' podman.registry = 'quay.io' diff --git a/tests/data/mock_pipeline_containers/per_profile_output/apple_container_containers.json b/tests/data/mock_pipeline_containers/per_profile_output/apple_container_containers.json new file mode 100644 index 0000000000..64ca108646 --- /dev/null +++ b/tests/data/mock_pipeline_containers/per_profile_output/apple_container_containers.json @@ -0,0 +1,11 @@ +{ + "MOCK_DOCKER_SINGLE_QUAY_IO": "quay.io/biocontainers/singlequay:1.9--pyh9f0ad1d_0", + "MOCK_DSL2_APPTAINER_VAR1": "quay.io/biocontainers/dsltwoapptainervarone:1.1.0--py38h7be5676_2", + "MOCK_DSL2_APPTAINER_VAR2": "quay.io/biocontainers/dsltwoapptainervartwo:1.1.0--hdfd78af_0", + "MOCK_DSL2_CURRENT": "quay.io/biocontainers/dsltwocurrent:1.2.1--pyhdfd78af_0", + "MOCK_DSL2_CURRENT_INVERTED": "quay.io/biocontainers/dsltwocurrentinv:3.3.2--h1b792b2_1", + "MOCK_DSL2_OLD": "quay.io/biocontainers/dsltwoold:0.23.0--0", + "MOCK_SEQERA_CONTAINER_HTTP": "community.wave.seqera.io/library/coreutils:9.5--ae99c88a9b28c264", + "MOCK_SEQERA_CONTAINER_ORAS": "community.wave.seqera.io/library/umi-transfer:1.0.0--d30e8812ea280fa1", + "MOCK_SEQERA_CONTAINER_ORAS_MULLED": "community.wave.seqera.io/library/umi-transfer_umicollapse:3298d4f1b49e33bd" +} diff --git a/tests/pipelines/__snapshots__/test_create_app/test_customisation_help.svg b/tests/pipelines/__snapshots__/test_create_app/test_customisation_help.svg index 90da8d5715..41adb4ba78 100644 --- a/tests/pipelines/__snapshots__/test_create_app/test_customisation_help.svg +++ b/tests/pipelines/__snapshots__/test_create_app/test_customisation_help.svg @@ -220,7 +220,7 @@ - + ⭘nf-core pipelines create — Create a new pipeline with the nf-core pipeline templa… @@ -237,8 +237,8 @@ ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ ▊▎Use a GitHub repository.Create a GitHub Show help  ▊▁▁▁▁▁▁▁▁▎repository for the▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -pipeline. -▇▇ +pipeline.▁▁ + ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ ▊▎Add Github badgesThe README.md file of Hide help  ▊▁▁▁▁▁▁▁▁▎the pipeline will▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ diff --git a/tests/pipelines/__snapshots__/test_create_app/test_type_custom.svg b/tests/pipelines/__snapshots__/test_create_app/test_type_custom.svg index bb027a6d32..24e6fbe196 100644 --- a/tests/pipelines/__snapshots__/test_create_app/test_type_custom.svg +++ b/tests/pipelines/__snapshots__/test_create_app/test_type_custom.svg @@ -236,7 +236,7 @@ ▊▎Use a GitHub repository.Create a GitHub Show help  ▊▁▁▁▁▁▁▁▁▎repository for the▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ pipeline. -▂▂ +▃▃ ▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ ▊▎Add Github badgesThe README.md file of Show help  ▊▁▁▁▁▁▁▁▁▎the pipeline will▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ diff --git a/tests/pipelines/__snapshots__/test_create_app/test_type_nfcore.svg b/tests/pipelines/__snapshots__/test_create_app/test_type_nfcore.svg index 2c6d37cfd5..1c2387d39d 100644 --- a/tests/pipelines/__snapshots__/test_create_app/test_type_nfcore.svg +++ b/tests/pipelines/__snapshots__/test_create_app/test_type_nfcore.svg @@ -214,7 +214,7 @@ - + ⭘nf-core pipelines create — Create a new pipeline with the nf-core pipeline templa… @@ -254,9 +254,9 @@ ▊▎Use GPUAdd GPU support to the Show help  ▊▁▁▁▁▁▁▁▁▎pipeline▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - - +▊▔▔▔▔▔▔▔▔▎▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +▊▎Use Apple ContainerAdd Apple Container Show help  +▊▁▁▁▁▁▁▁▁▎support to the pipeline▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ diff --git a/tests/pipelines/download/test_apple_container.py b/tests/pipelines/download/test_apple_container.py new file mode 100644 index 0000000000..1f024797bc --- /dev/null +++ b/tests/pipelines/download/test_apple_container.py @@ -0,0 +1,198 @@ +"""Tests for the Apple Container fetcher in nf-core download.""" + +import os +import shutil +import unittest +from contextlib import redirect_stderr +from io import StringIO +from pathlib import Path +from unittest import mock + +import pytest + +from nf_core.pipelines.download.apple_container import AppleContainerFetcher + +from ...utils import with_temporary_folder + + +class AppleContainerTest(unittest.TestCase): + @pytest.fixture(autouse=True) + def use_caplog(self, caplog): + self._caplog = caplog + + # + # Test for AppleContainerFetcher initialization and output directory + # + @pytest.mark.skipif( + shutil.which("container") is None, + reason="Can't test Apple Container fetcher without Apple Container CLI installed.", + ) + @with_temporary_folder + def test_apple_container_output_dir(self, tmp_path): + tmp_path = Path(tmp_path) + fetcher = AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=[], + ) + assert fetcher.get_container_output_dir() == tmp_path / "apple-container-images" + + # + # Test for AppleContainerFetcher._write_apple_container_load_message + # + @pytest.mark.skipif( + shutil.which("container") is None, + reason="Can't test Apple Container fetcher without Apple Container CLI installed.", + ) + @with_temporary_folder + def test_apple_container_write_load_message(self, tmp_path): + tmp_path = Path(tmp_path) + fetcher = AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=[], + ) + img_dir = fetcher.get_container_output_dir() + img_dir.mkdir() + with redirect_stderr(StringIO()) as f: + fetcher._write_apple_container_load_message() + + output = f.getvalue() + # Check that the message references correct scripts + # Rich may insert hard line breaks in long paths, so strip all whitespace for path check + assert "apple-container-images" in output.replace("\n", "").replace(" ", "") + assert "appleContainer-load.sh" in output + + # Check that the script was written and is executable + assert (img_dir / "appleContainer-load.sh").exists() + assert os.access(img_dir / "appleContainer-load.sh", os.X_OK) + + # + # Test that cleanup calls the Apple Container message, not the Docker one + # + @pytest.mark.skipif( + shutil.which("container") is None, + reason="Can't test Apple Container fetcher without Apple Container CLI installed.", + ) + @with_temporary_folder + def test_apple_container_cleanup(self, tmp_path): + tmp_path = Path(tmp_path) + fetcher = AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=[], + ) + img_dir = fetcher.get_container_output_dir() + img_dir.mkdir() + with redirect_stderr(StringIO()) as f: + fetcher.cleanup() + + output = f.getvalue() + # Should mention Apple Container, not Docker/podman + assert "appleContainer-load.sh" in output + # Should NOT have the Docker-specific "podman-load.sh (experimental)" wording + assert "experimental" not in output + assert "podman" not in output + + # + # Test that gather_registries includes appleContainer.registry key + # + @pytest.mark.skipif( + shutil.which("container") is None, + reason="Can't test Apple Container fetcher without Apple Container CLI installed.", + ) + @with_temporary_folder + @mock.patch("nf_core.pipelines.download.container_fetcher.ContainerFetcher.gather_config_registries") + def test_apple_container_gather_registries(self, tmp_path, mock_gather_config): + tmp_path = Path(tmp_path) + mock_gather_config.return_value = {"quay.io"} + fetcher = AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=["docker.io"], + ) + registries = fetcher.gather_registries(tmp_path) + # Should have called gather_config_registries with the apple container key + mock_gather_config.assert_called_once_with( + tmp_path, + ["appleContainer.registry", "docker.registry", "podman.registry"], + ) + assert "quay.io" in registries + assert "docker.io" in registries + + # + # Test that check_and_set_implementation finds the container binary + # + @pytest.mark.skipif( + shutil.which("container") is None, + reason="Can't test Apple Container fetcher without Apple Container CLI installed.", + ) + @with_temporary_folder + def test_apple_container_check_implementation(self, tmp_path): + tmp_path = Path(tmp_path) + fetcher = AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=[], + ) + assert fetcher.implementation == "container" + + # + # Test that check_and_set_implementation raises when container CLI is missing + # + @with_temporary_folder + @mock.patch("shutil.which", return_value=None) + def test_apple_container_missing_cli_raises(self, tmp_path, mock_which): + tmp_path = Path(tmp_path) + with pytest.raises(OSError, match="Apple Container CLI"): + AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=[], + ) + + # + # Test that construct_pull_command produces the correct command + # + @pytest.mark.skipif( + shutil.which("container") is None, + reason="Can't test Apple Container fetcher without Apple Container CLI installed.", + ) + @with_temporary_folder + def test_apple_container_pull_command(self, tmp_path): + tmp_path = Path(tmp_path) + fetcher = AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=[], + ) + cmd = fetcher.construct_pull_command("quay.io/biocontainers/fastqc:0.12.1") + assert cmd == ["container", "image", "pull", "--platform", "linux/amd64", "quay.io/biocontainers/fastqc:0.12.1"] + + # + # Test that construct_save_command produces the correct command + # + @pytest.mark.skipif( + shutil.which("container") is None, + reason="Can't test Apple Container fetcher without Apple Container CLI installed.", + ) + @with_temporary_folder + def test_apple_container_save_command(self, tmp_path): + tmp_path = Path(tmp_path) + fetcher = AppleContainerFetcher( + outdir=tmp_path, + container_library=[], + registry_set=[], + ) + output_path = tmp_path / "image.tar" + cmd = fetcher.construct_save_command(output_path, "quay.io/biocontainers/fastqc:0.12.1") + assert cmd == [ + "container", + "image", + "save", + "--platform", + "linux/amd64", + "--output", + str(output_path), + "quay.io/biocontainers/fastqc:0.12.1", + ] diff --git a/tests/pipelines/download/test_download.py b/tests/pipelines/download/test_download.py index 092817b067..e91ccc5132 100644 --- a/tests/pipelines/download/test_download.py +++ b/tests/pipelines/download/test_download.py @@ -353,6 +353,42 @@ def test_containers_pipeline_docker(self, tmp_path, mock_fetch_wf_config): f"Containers that should've been found: {ref_container_strs}" ) + # + # Test that `find_container_images` (uses `nextflow inspect`) fetches the correct Apple Container images + # + @pytest.mark.skipif( + shutil.which("nextflow") is None or not check_nextflow_version(NF_INSPECT_MIN_NF_VERSION), + reason=f"Can't run test that requires Nextflow >= {NF_INSPECT_MIN_NF_VERSION} to run if not installed.", + ) + @with_temporary_folder + @mock.patch("nf_core.utils.fetch_wf_config") + def test_containers_pipeline_apple_container(self, tmp_path, mock_fetch_wf_config): + tmp_path = Path(tmp_path) + assert check_nextflow_version(NF_INSPECT_MIN_NF_VERSION) is True + + # Set up test + container_system = "apple_container" + mock_pipeline_dir = TEST_DATA_DIR / "mock_pipeline_containers" + refererence_json_dir = mock_pipeline_dir / "per_profile_output" + # Check that `-profile apple_container` produces the same output as the reference + download_obj = DownloadWorkflow(pipeline="dummy", outdir=tmp_path, container_system=container_system) + mock_fetch_wf_config.return_value = {} + + # NF >= 26.04 rejects old-style if/else container directives (mock_dsl2_old uses them) + entrypoint = "main_passing_test.nf" + with self._strict_syntax_ctx(match="downgrade to Nextflow"): + download_obj.find_container_images(mock_pipeline_dir, "dummy-revision", entrypoint=entrypoint) + + if not self.nf_strict_syntax: + found_containers = set(download_obj.containers) + with open(refererence_json_dir / f"{container_system}_containers.json") as fh: + ref_containers = json.load(fh) + ref_container_strs = set(ref_containers.values()) + assert found_containers == ref_container_strs, ( + f"Containers found in pipeline by `nextflow inspect`: {found_containers}\n" + f"Containers that should've been found: {ref_container_strs}" + ) + @mock.patch("nf_core.pipelines.download.download.run_cmd") @mock.patch("nf_core.pipelines.list.Workflows.get_remote_workflows") def test_find_container_images_retries_with_outdir_on_missing_param_error(