feat(panel): store concatenated panels and bump each source on read

src/pixelator/pna/analysis/denoise.py

@@ -20,7 +20,7 @@
 from pixelator.pna.analysis_engine import PerComponentTask
 from pixelator.pna.anndata import add_missing_adata_info, pna_edgelist_to_anndata
 from pixelator.pna.config import pna_config
-from pixelator.pna.config.panel import PNAAntibodyPanel, load_antibody_panel
+from pixelator.pna.config.panel import load_antibody_panel
 from pixelator.pna.graph import PNAGraph
 from pixelator.pna.graph.adaptive_core_expansion import adaptive_core_expansion
 from pixelator.pna.graph.node_pls import (
@@ -29,7 +29,7 @@
     node_pls,
 )
 from pixelator.pna.pixeldataset import PNAPixelDataset, read
-from pixelator.pna.pixeldataset.io import PixelFileWriter, PxlFile
+from pixelator.pna.pixeldataset.io import PixelFileWriter, PxlFile, read_dataset_panel
 
 logger = logging.getLogger(__name__)
 
@@ -645,7 +645,7 @@ def add_to_pixel_file(self, data: pd.DataFrame, pxl_file_target: PxlFile):
         pxl = PNAPixelDataset.from_files(pxl_file_target)
         old_adata = pxl.adata()
         try:
-            panel = PNAAntibodyPanel.from_pxl_dataset(read(pxl_file_target.path))
+            panel = read_dataset_panel(read(pxl_file_target.path))
         except KeyError:
             # If pxl file does not contain panel data, try to load it from
             # the panel name.
@@ -662,7 +662,14 @@ def add_to_pixel_file(self, data: pd.DataFrame, pxl_file_target: PxlFile):
             write_denoised_edgelist(pxl, nodes_to_remove, denoised_edgelist_path)
             with PixelFileWriter(pxl_file_target.path) as writer:
                 writer.write_edgelist(Path(denoised_edgelist_path))
-                adata = pna_edgelist_to_anndata(writer.get_connection(), panel)
+                # Rebuild counts from the markers already in var. Using the full
+                # panel would put hashing clones removed by sample calling back.
+                adata = pna_edgelist_to_anndata(
+                    writer.get_connection(),
+                    panel,
+                    markers=list(old_adata.var_names),
+                )
+                writer.write_panel(panel)
                 old_adata.obs.rename(
                     columns={"isotype_fraction": "pre_denoise_isotype_fraction"},
                     inplace=True,

src/pixelator/pna/anndata.py

@@ -30,25 +30,20 @@ def calculate_antibody_metrics(counts_df):
     return pd.concat([total_antibody, relative_antibody, components_detected], axis=1)
 
 
-def add_panel_information(adata: AnnData, panel: PNAAntibodyPanel) -> AnnData:
-    """Add panel data to var."""
-    adata.var = adata.var.join(panel.df, how="left")
-
-    adata.uns["panel_metadata"] = panel.metadata.model_dump()
-    adata.uns["panel_metadata"]["panel_columns"] = list(panel.df.columns)
-
-    return adata
-
-
 def pna_edgelist_to_anndata(
-    pixel_connection: duckdb.DuckDBPyConnection, panel: PNAAntibodyPanel
+    pixel_connection: duckdb.DuckDBPyConnection,
+    panel: PNAAntibodyPanel,
+    markers: list[str] | None = None,
 ) -> AnnData:
     """Build an AnnData object from a DuckDB connection to a pixel file and a panel object.
 
     Args:
         pixel_connection: A DuckDB connection to a pixel file. The connection must contain an 'edgelist' table
             with the required columns (e.g., component, marker_1, marker_2, umi1, umi2, read_count).
         panel: The antibody panel object containing marker metadata.
+        markers: Marker ids to use as the count-matrix columns. Defaults to
+            every marker on ``panel``. Denoise passes the markers already in
+            ``var`` so hashing clones removed by sample calling are not added back.
 
     Returns:
         An AnnData object with counts and panel information.
@@ -73,10 +68,11 @@ def pna_edgelist_to_anndata(
         .tolist()
     )
 
+    marker_list = list(panel.markers if markers is None else markers)
     n_components = len(components)
-    n_markers = len(panel.markers)
+    n_markers = len(marker_list)
     component_to_idx = {c: i for i, c in enumerate(components)}
-    marker_to_idx = {m: i for i, m in enumerate(panel.markers)}
+    marker_to_idx = {m: i for i, m in enumerate(marker_list)}
 
     X = np.zeros((n_components, n_markers), dtype=np.uint32)
     n_umi1_arr = np.zeros(n_components, dtype=np.uint64)
@@ -152,7 +148,7 @@ def pna_edgelist_to_anndata(
     node_counts_df = pd.DataFrame(
         X,
         index=component_index,
-        columns=pd.Index(panel.markers, name="marker_id"),
+        columns=pd.Index(marker_list, name="marker_id"),
     )
 
     logger.debug("Computing component metrics.")
@@ -182,7 +178,7 @@ def pna_edgelist_to_anndata(
 
     logger.debug("Computing antibody metrics.")
     antibody_metrics_df = calculate_antibody_metrics(counts_df=node_counts_df)
-    antibody_metrics_df = antibody_metrics_df.reindex(index=panel.markers, fill_value=0)
+    antibody_metrics_df = antibody_metrics_df.reindex(index=marker_list, fill_value=0)
     antibody_metrics_df.index.name = "marker_id"
     # Do a dtype conversion of the columns here since AnnData cannot handle
     # a pyarrow arrays.
@@ -198,10 +194,17 @@ def pna_edgelist_to_anndata(
         var=antibody_metrics_df,
     )
 
-    adata = add_panel_information(adata, panel)
-
     total_marker_counts = node_counts_df.sum(axis=1)
-    isotype_markers = adata.var[adata.var["control"]].index
+    control = panel.df["control"]
+    if pd.api.types.is_bool_dtype(control):
+        control_mask = control.fillna(False).astype(bool)
+    else:
+        control_mask = control.astype(str).str.lower().eq("yes")
+    isotype_markers = [
+        marker
+        for marker in control_mask.index[control_mask]
+        if marker in node_counts_df.columns
+    ]
     isotype_counts = node_counts_df[isotype_markers].sum(axis=1)
     adata.obs["isotype_fraction"] = isotype_counts / total_marker_counts
 

src/pixelator/pna/cli/collapse.py

@@ -29,7 +29,8 @@
 )
 from pixelator.pna.collapse.paired.collapser import MoleculeCollapser
 from pixelator.pna.collapse.utilities import split_collapse_inputs
-from pixelator.pna.config import load_antibody_panel, pna_config
+from pixelator.pna.config import pna_config
+from pixelator.pna.config.panel import load_antibody_panels
 from pixelator.pna.utils import get_demux_filename_info
 
 logger = logging.getLogger("collapse")
@@ -124,7 +125,7 @@ def collapse(
     sanity_check_inputs(input_files=input_files, allowed_extensions=("parquet",))
 
     assay = pna_config.get_assay(design)
-    panel = load_antibody_panel(pna_config, panel)
+    panel = load_antibody_panels(pna_config, panel)
 
     # create the output directory
     collapse_output = create_output_stage_dir(output, "collapse")

src/pixelator/pna/cli/common.py

@@ -103,16 +103,7 @@ def wrapper(*args, **kwargs):
     return wrapper
 
 
-def validate_panel(ctx, param, value):
-    """Validate the panel commandline option.
-
-    Args:
-        ctx: The click context
-        param: The click parameter
-        value: The click value
-    Returns:
-        The validated value
-    """
+def _validate_one_panel(value: str) -> str:
     try:
         if Path(value).exists():
             return value
@@ -133,6 +124,26 @@ def validate_panel(ctx, param, value):
     return value
 
 
+def validate_panel(ctx, param, value):
+    """Validate the panel commandline option.
+
+    ``--panel`` may be repeated. Each value is a supported panel name or a path
+    to a panel CSV.
+
+    Args:
+        ctx: The click context
+        param: The click parameter
+        value: The click value
+    Returns:
+        The validated value
+    """
+    if value is None:
+        return value
+    if isinstance(value, str):
+        return _validate_one_panel(value)
+    return tuple(_validate_one_panel(item) for item in value)
+
+
 def panel_option(func):
     """Decorate a click command and add the --panel option."""
     from pixelator.pna.config import pna_config
@@ -144,10 +155,13 @@ def panel_option(func):
     @click.option(
         "--panel",
         required=True,
-        default=None,
+        multiple=True,
         type=click.UNPROCESSED,
         callback=validate_panel,
-        help="The name of a panel to load from the supported panels. Optionally, provide a path to a custom panel file.",
+        help=(
+            "The name of a panel to load from the supported panels, or a path to a "
+            "custom panel file. Repeat to concatenate several panels."
+        ),
     )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):

src/pixelator/pna/cli/demux.py

@@ -25,7 +25,7 @@
     threads_option,
 )
 from pixelator.pna.config import pna_config
-from pixelator.pna.config.panel import load_antibody_panel
+from pixelator.pna.config.panel import load_antibody_panels
 from pixelator.pna.demux import (
     correct_marker_barcodes,
     demux_barcode_groups,
@@ -143,7 +143,7 @@ def demux(
     # load assay design
     assay = pna_config.get_assay(design)
     # load marker panel
-    panel = load_antibody_panel(pna_config, panel)
+    panel = load_antibody_panels(pna_config, panel)
 
     logger.info(f"Correcting marker barcodes for input: {fastq_file}")
 

src/pixelator/pna/cli/graph.py

@@ -18,7 +18,8 @@
     write_parameters_file,
 )
 from pixelator.pna.cli.common import logger, output_option, panel_option
-from pixelator.pna.config import load_antibody_panel, pna_config
+from pixelator.pna.config import pna_config
+from pixelator.pna.config.panel import load_antibody_panels
 from pixelator.pna.graph.community_detection import (
     RefinementOptions,
     StagedRefinementOptions,
@@ -256,7 +257,7 @@ def graph(
     )
     output_path = graph_output / f"{sample_name}.graph.pxl"
 
-    panel = load_antibody_panel(pna_config, panel)
+    panel = load_antibody_panels(pna_config, panel)
     initial_stage_refinement_options = RefinementOptions(
         leiden_resolution=initial_stage_leiden_resolution,
         max_edges_to_remove=initial_stage_max_edges_to_remove,

src/pixelator/pna/cli/sample_calling.py

@@ -18,7 +18,7 @@
 )
 from pixelator.pna import read
 from pixelator.pna.cli.common import output_option
-from pixelator.pna.config.panel import PNAAntibodyPanel
+from pixelator.pna.pixeldataset.io import read_dataset_panel
 from pixelator.pna.sample_calling import (
     create_final_report,
     sample_calling,
@@ -99,10 +99,14 @@ def sample_calling_cli(
     pool_name = Path(input_pxl_file).name.split(".")[0]
     undetermined_sample_name = f"{pool_name}_undetermined"
 
-    panel_info = PNAAntibodyPanel.from_pxl_dataset(read(input_pxl_file))
-    hashing_antibodies_in_panel = set(
-        panel_info.df[panel_info.df["sample_hashing"] == "yes"].index.to_list()
-    )
+    panel = read_dataset_panel(read(input_pxl_file))
+    if "sample_hashing" not in panel.df.columns:
+        raise ValueError(
+            "Sample calling requires a sample_hashing column on the panel "
+            "so hashing markers can be identified. This panel has no "
+            "sample_hashing column."
+        )
+    hashing_antibodies_in_panel = panel.hashing_marker_ids
     samplesheet_df = pl.read_csv(samplesheet)
     if "undetermined" in samplesheet_df["sample"].to_list():
         raise ValueError(

src/pixelator/pna/config/__init__.py

@@ -19,6 +19,7 @@
 from pixelator.pna.config.panel import (
     PNAAntibodyPanel,
     load_antibody_panel,
+    load_antibody_panels,
 )
 
 __all__ = [
@@ -35,4 +36,5 @@
     "PNAAntibodyPanel",
     "AntibodyPanelMetadata",
     "load_antibody_panel",
+    "load_antibody_panels",
 ]

src/pixelator/pna/config/config_class.py

@@ -93,16 +93,23 @@ def load_panel_file(self, path: PathType) -> None:
     def add_panel(self, panel: PNAAntibodyPanel) -> None:
         """Register a panel and its lookup keys in the config.
 
-        The panel is indexed by panel name (or filename fallback), optional product,
-        and aliases.
+        The panel is indexed by panel name, optional product, and aliases.
+        Only a single-source panel can be registered. Panel CSV files are
+        one source.
 
         Args:
             panel: Panel object to add.
 
         Raises:
+            ValueError: If ``panel`` has more or fewer than one source.
             PanelException: If an alias already maps to a different panel key.
         """
-        key = panel.name if panel.name is not None else str(panel.filename)
+        if len(panel.sources) != 1:
+            raise ValueError(
+                "Only a single-source panel can be added to the config. "
+                f"Got {len(panel.sources)} sources."
+            )
+        key = panel.name
         self.panels[key].append(panel)
 
         # allow to also get panel by product name if provided in the panel file
@@ -279,18 +286,17 @@ def get_panel(
                 + "alias to disambiguate.",
             )
 
-        def keyfunc(p):
+        def keyfunc(p: PNAAntibodyPanel):
             """Keyfunc.
 
             Args:
-                p: p.
+                p: PNAAntibodyPanel.
+
+            Returns:
+                Parsed semantic version of the panel.
             """
             version = p.version
-            if version is None:
-                v = semver.Version.parse("0.0.0")
-
-            v = semver.Version.parse(version)
-            return v
+            return semver.Version.parse(version)
 
         panels_with_key = sorted(panels_with_key, key=keyfunc, reverse=True)
         return panels_with_key[0]