feat(panel): store concatenated panels and bump each source on read

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+- `single-cell-pna demux`, `collapse`, and `graph` accept repeated `--panel` options. The panels are concatenated, and `marker_id`, `sequence_1`, and `sequence_2` must be unique across the result.
 - `pixelator.pna.analysis.differential_abundance` to run marker differential abundance (Wilcoxon via `scanpy.tl.rank_genes_groups`). Effect size is a mean difference, not scanpy's log-fold change.
 - Samples with no cells above the size threshold no longer fail the pipeline. A null pxl file
   with a reason is written and passed through later steps, so the sample still shows up downstream.
@@ -26,6 +27,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `pixelator.pna.analysis.filter_proximity_scores` to filter a proximity score table by marker abundance (Python analog of pixelatorR `FilterProximityScores`).
 
 ### Changed
+- New `.pxl` files store the antibody panel in the `panels` and `panel_sources` tables. Panel columns are joined onto the AnnData in memory when the file is read, and are not written to `var`. `uns['panel_metadata']` is no longer written, and pixel-file metadata no longer includes `panel_name` or `panel_version`.
+- When several `.pxl` files are read together, each panel source is updated to the newest patch those files carry, matching on name, product, and major.minor. A source that is not already on a file is left alone.
+- `PNAConfig.add_panel` only accepts a panel with a single source.
 - Updated pixelgen-pixelator-core to 0.2.0 improving peak memory usage in the graph step by ~20%.
 - `density_scatter_plot` now lives in `pixelator.plot` (previously `pixelator.mpx.plot`).
 - `uei_count` is now optional on PNA edgelists in `sample_calling` and the graph component
@@ -38,6 +42,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   when one exists.
 
 ### Removed
+- `PNAAntibodyPanel.from_adata`. Panels are read from the `.pxl` file. Files from pixelator 0.22.0 through 0.30.0 are still read from `uns['panel_metadata']` and `var`. Denoise still uses `panel_name` for files from before 0.22.0.
 - Stale nf-core/pixelator integration tests (`task test-nf-core-pixelator` and
   `.github/workflows/nf-core-pixelator-tests.yml`).
 - The `fruchterman_reingold`, `fruchterman_reingold_3d`, `kamada_kawai`, and
@@ -54,6 +59,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   passed directly to native community detection, and the recovered components are unchanged.
 
 ### Fixed
+- A panel patch bump renames markers in the edgelist, proximity, and layouts, not only in `var`.
+- A panel patch bump of a sample-called file no longer fails because hashing clones are missing from `var`. A missing non-hashing marker still fails.
+- Denoise rebuilds the count matrix from the markers already in `var`, so hashing clones removed by sample calling do not come back. The full panel is still stored in the panel tables.
 - `coarsened_pmds_layout` sizes PMDS pivots from the full graph when Leiden
   yields too few communities, so a valid low `pivots` no longer fails
   `pmds_layout`'s `0.2 * n` lower bound.

src/pixelator/pna/analysis/denoise.py

@@ -20,7 +20,7 @@
 from pixelator.pna.analysis_engine import PerComponentTask
 from pixelator.pna.anndata import add_missing_adata_info, pna_edgelist_to_anndata
 from pixelator.pna.config import pna_config
-from pixelator.pna.config.panel import PNAAntibodyPanel, load_antibody_panel
+from pixelator.pna.config.panel import load_antibody_panel
 from pixelator.pna.graph import PNAGraph
 from pixelator.pna.graph.adaptive_core_expansion import adaptive_core_expansion
 from pixelator.pna.graph.node_pls import (
@@ -29,7 +29,7 @@
     node_pls,
 )
 from pixelator.pna.pixeldataset import PNAPixelDataset, read
-from pixelator.pna.pixeldataset.io import PixelFileWriter, PxlFile
+from pixelator.pna.pixeldataset.io import PixelFileWriter, PxlFile, read_dataset_panel
 
 logger = logging.getLogger(__name__)
 
@@ -645,7 +645,7 @@ def add_to_pixel_file(self, data: pd.DataFrame, pxl_file_target: PxlFile):
         pxl = PNAPixelDataset.from_files(pxl_file_target)
         old_adata = pxl.adata()
         try:
-            panel = PNAAntibodyPanel.from_pxl_dataset(read(pxl_file_target.path))
+            panel = read_dataset_panel(read(pxl_file_target.path))
         except KeyError:
             # If pxl file does not contain panel data, try to load it from
             # the panel name.
@@ -662,7 +662,14 @@ def add_to_pixel_file(self, data: pd.DataFrame, pxl_file_target: PxlFile):
             write_denoised_edgelist(pxl, nodes_to_remove, denoised_edgelist_path)
             with PixelFileWriter(pxl_file_target.path) as writer:
                 writer.write_edgelist(Path(denoised_edgelist_path))
-                adata = pna_edgelist_to_anndata(writer.get_connection(), panel)
+                # Rebuild counts from the markers already in var. Using the full
+                # panel would put hashing clones removed by sample calling back.
+                adata = pna_edgelist_to_anndata(
+                    writer.get_connection(),
+                    panel,
+                    markers=list(old_adata.var_names),
+                )
+                writer.write_panel(panel)
                 old_adata.obs.rename(
                     columns={"isotype_fraction": "pre_denoise_isotype_fraction"},
                     inplace=True,

src/pixelator/pna/anndata.py

@@ -30,25 +30,20 @@ def calculate_antibody_metrics(counts_df):
     return pd.concat([total_antibody, relative_antibody, components_detected], axis=1)
 
 
-def add_panel_information(adata: AnnData, panel: PNAAntibodyPanel) -> AnnData:
-    """Add panel data to var."""
-    adata.var = adata.var.join(panel.df, how="left")
-
-    adata.uns["panel_metadata"] = panel.metadata.model_dump()
-    adata.uns["panel_metadata"]["panel_columns"] = list(panel.df.columns)
-
-    return adata
-
-
 def pna_edgelist_to_anndata(
-    pixel_connection: duckdb.DuckDBPyConnection, panel: PNAAntibodyPanel
+    pixel_connection: duckdb.DuckDBPyConnection,
+    panel: PNAAntibodyPanel,
+    markers: list[str] | None = None,
 ) -> AnnData:
     """Build an AnnData object from a DuckDB connection to a pixel file and a panel object.
 
     Args:
         pixel_connection: A DuckDB connection to a pixel file. The connection must contain an 'edgelist' table
             with the required columns (e.g., component, marker_1, marker_2, umi1, umi2, read_count).
         panel: The antibody panel object containing marker metadata.
+        markers: Marker ids to use as the count-matrix columns. Defaults to
+            every marker on ``panel``. Denoise passes the markers already in
+            ``var`` so hashing clones removed by sample calling are not added back.
 
     Returns:
         An AnnData object with counts and panel information.
@@ -73,10 +68,11 @@ def pna_edgelist_to_anndata(
         .tolist()
     )
 
+    marker_list = list(panel.markers if markers is None else markers)
     n_components = len(components)
-    n_markers = len(panel.markers)
+    n_markers = len(marker_list)
     component_to_idx = {c: i for i, c in enumerate(components)}
-    marker_to_idx = {m: i for i, m in enumerate(panel.markers)}
+    marker_to_idx = {m: i for i, m in enumerate(marker_list)}
 
     X = np.zeros((n_components, n_markers), dtype=np.uint32)
     n_umi1_arr = np.zeros(n_components, dtype=np.uint64)
@@ -152,7 +148,7 @@ def pna_edgelist_to_anndata(
     node_counts_df = pd.DataFrame(
         X,
         index=component_index,
-        columns=pd.Index(panel.markers, name="marker_id"),
+        columns=pd.Index(marker_list, name="marker_id"),
     )
 
     logger.debug("Computing component metrics.")
@@ -182,7 +178,7 @@ def pna_edgelist_to_anndata(
 
     logger.debug("Computing antibody metrics.")
     antibody_metrics_df = calculate_antibody_metrics(counts_df=node_counts_df)
-    antibody_metrics_df = antibody_metrics_df.reindex(index=panel.markers, fill_value=0)
+    antibody_metrics_df = antibody_metrics_df.reindex(index=marker_list, fill_value=0)
     antibody_metrics_df.index.name = "marker_id"
     # Do a dtype conversion of the columns here since AnnData cannot handle
     # a pyarrow arrays.
@@ -198,10 +194,17 @@ def pna_edgelist_to_anndata(
         var=antibody_metrics_df,
     )
 
-    adata = add_panel_information(adata, panel)
-
     total_marker_counts = node_counts_df.sum(axis=1)
-    isotype_markers = adata.var[adata.var["control"]].index
+    control = panel.df["control"]
+    if pd.api.types.is_bool_dtype(control):
+        control_mask = control.fillna(False).astype(bool)
+    else:
+        control_mask = control.astype(str).str.lower().eq("yes")
+    isotype_markers = [
+        marker
+        for marker in control_mask.index[control_mask]
+        if marker in node_counts_df.columns
+    ]
     isotype_counts = node_counts_df[isotype_markers].sum(axis=1)
     adata.obs["isotype_fraction"] = isotype_counts / total_marker_counts
 

src/pixelator/pna/cli/collapse.py

@@ -29,7 +29,8 @@
 )
 from pixelator.pna.collapse.paired.collapser import MoleculeCollapser
 from pixelator.pna.collapse.utilities import split_collapse_inputs
-from pixelator.pna.config import load_antibody_panel, pna_config
+from pixelator.pna.config import pna_config
+from pixelator.pna.config.panel import load_antibody_panels
 from pixelator.pna.utils import get_demux_filename_info
 
 logger = logging.getLogger("collapse")
@@ -124,7 +125,7 @@ def collapse(
     sanity_check_inputs(input_files=input_files, allowed_extensions=("parquet",))
 
     assay = pna_config.get_assay(design)
-    panel = load_antibody_panel(pna_config, panel)
+    panel = load_antibody_panels(pna_config, panel)
 
     # create the output directory
     collapse_output = create_output_stage_dir(output, "collapse")

src/pixelator/pna/cli/common.py

@@ -5,6 +5,7 @@
 
 import functools
 import logging
+import os
 from pathlib import Path
 
 import click
@@ -103,16 +104,7 @@ def wrapper(*args, **kwargs):
     return wrapper
 
 
-def validate_panel(ctx, param, value):
-    """Validate the panel commandline option.
-
-    Args:
-        ctx: The click context
-        param: The click parameter
-        value: The click value
-    Returns:
-        The validated value
-    """
+def _validate_one_panel(value: str) -> str:
     try:
         if Path(value).exists():
             return value
@@ -133,6 +125,26 @@ def validate_panel(ctx, param, value):
     return value
 
 
+def validate_panel(ctx, param, value):
+    """Validate the panel commandline option.
+
+    ``--panel`` may be repeated. Each value is a supported panel name or a path
+    to a panel CSV.
+
+    Args:
+        ctx: The click context
+        param: The click parameter
+        value: The click value
+    Returns:
+        The validated value
+    """
+    if value is None:
+        return value
+    if isinstance(value, (str, os.PathLike)):
+        return _validate_one_panel(value)
+    return tuple(_validate_one_panel(item) for item in value)
+
+
 def panel_option(func):
     """Decorate a click command and add the --panel option."""
     from pixelator.pna.config import pna_config
@@ -144,10 +156,13 @@ def panel_option(func):
     @click.option(
         "--panel",
         required=True,
-        default=None,
+        multiple=True,
         type=click.UNPROCESSED,
         callback=validate_panel,
-        help="The name of a panel to load from the supported panels. Optionally, provide a path to a custom panel file.",
+        help=(
+            "The name of a panel to load from the supported panels, or a path to a "
+            "custom panel file. Repeat to concatenate several panels."
+        ),
     )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):

src/pixelator/pna/cli/demux.py

@@ -25,7 +25,7 @@
     threads_option,
 )
 from pixelator.pna.config import pna_config
-from pixelator.pna.config.panel import load_antibody_panel
+from pixelator.pna.config.panel import load_antibody_panels
 from pixelator.pna.demux import (
     correct_marker_barcodes,
     demux_barcode_groups,
@@ -143,7 +143,7 @@ def demux(
     # load assay design
     assay = pna_config.get_assay(design)
     # load marker panel
-    panel = load_antibody_panel(pna_config, panel)
+    panel = load_antibody_panels(pna_config, panel)
 
     logger.info(f"Correcting marker barcodes for input: {fastq_file}")
 

src/pixelator/pna/cli/graph.py

@@ -18,7 +18,8 @@
     write_parameters_file,
 )
 from pixelator.pna.cli.common import logger, output_option, panel_option
-from pixelator.pna.config import load_antibody_panel, pna_config
+from pixelator.pna.config import pna_config
+from pixelator.pna.config.panel import load_antibody_panels
 from pixelator.pna.graph.community_detection import (
     RefinementOptions,
     StagedRefinementOptions,
@@ -257,7 +258,7 @@ def graph(
     )
     output_path = graph_output / f"{sample_name}.graph.pxl"
 
-    panel = load_antibody_panel(pna_config, panel)
+    panel = load_antibody_panels(pna_config, panel)
     initial_stage_refinement_options = RefinementOptions(
         leiden_resolution=initial_stage_leiden_resolution,
         max_edges_to_remove=initial_stage_max_edges_to_remove,
@@ -316,8 +317,6 @@ def graph(
             output_path,
             sample_name=sample_name,
             reason=str(exc),
-            panel_name=panel.name,
-            panel_version=panel.version,
         )
         report = _graph_report(
             sample_name,

src/pixelator/pna/cli/sample_calling.py

@@ -19,9 +19,8 @@
 )
 from pixelator.pna import read
 from pixelator.pna.cli.common import output_option
-from pixelator.pna.config.panel import PNAAntibodyPanel
 from pixelator.pna.pixeldataset import NullPxlFileError
-from pixelator.pna.pixeldataset.io import PxlFile, write_null_pxl
+from pixelator.pna.pixeldataset.io import PxlFile, read_dataset_panel, write_null_pxl
 from pixelator.pna.sample_calling import (
     create_final_report,
     sample_calling,
@@ -106,7 +105,7 @@ def sample_calling_cli(
     undetermined_sample_name = f"{pool_name}_undetermined"
 
     try:
-        panel_info = PNAAntibodyPanel.from_pxl_dataset(read(input_pxl_file))
+        panel = read_dataset_panel(read(input_pxl_file))
     except NullPxlFileError as exc:
         logger.warning("%s", exc)
         _pass_through_null_sample_calling(
@@ -118,9 +117,13 @@ def sample_calling_cli(
             pool_metadata=PxlFile(Path(input_pxl_file)).metadata(),
         )
         return
-    hashing_antibodies_in_panel = set(
-        panel_info.df[panel_info.df["sample_hashing"] == "yes"].index.to_list()
-    )
+    if "sample_hashing" not in panel.df.columns:
+        raise ValueError(
+            "Sample calling requires a sample_hashing column on the panel "
+            "so hashing markers can be identified. This panel has no "
+            "sample_hashing column."
+        )
+    hashing_antibodies_in_panel = panel.hashing_marker_ids
     samplesheet_df = pl.read_csv(samplesheet)
     _reject_reserved_samplesheet_names(
         samplesheet_df["sample"].to_list(),

src/pixelator/pna/config/__init__.py

@@ -19,6 +19,7 @@
 from pixelator.pna.config.panel import (
     PNAAntibodyPanel,
     load_antibody_panel,
+    load_antibody_panels,
 )
 
 __all__ = [
@@ -35,4 +36,5 @@
     "PNAAntibodyPanel",
     "AntibodyPanelMetadata",
     "load_antibody_panel",
+    "load_antibody_panels",
 ]