Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pyhealth/metrics/interpretability/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -453,10 +453,10 @@ def compute(
)

# Compute probability drop
original_class_probs = y_probs
original_class_probs = y_probs.clone()
original_class_probs[neg_mask] = -original_class_probs[neg_mask]

ablated_class_probs = ablated_probs
ablated_class_probs = ablated_probs.clone()
ablated_class_probs[neg_mask] = -ablated_class_probs[neg_mask]

prob_drop = torch.zeros(batch_size, device=y_probs.device)
Expand Down
27 changes: 27 additions & 0 deletions tests/core/test_interp_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from pyhealth.metrics.interpretability import (
ComprehensivenessMetric,
Evaluator,
SampleClass,
SufficiencyMetric,
threshold_sample_filter,
)
Expand Down Expand Up @@ -449,6 +450,32 @@ def test_percentage_sensitivity(self):
self.assertTrue(torch.isfinite(torch.tensor(score_10)))
self.assertTrue(torch.isfinite(torch.tensor(score_50)))

def test_negative_class_scores_independent_of_percentage_order(self):
"""Test that a negative-class sample's score at a percentage is order-independent."""
attributions = self._create_attributions(self.batch)

def negative_filter(y_probs, classifier_type):
return torch.full(
(y_probs.shape[0],),
SampleClass.NEGATIVE,
dtype=torch.long,
device=y_probs.device,
)

def score_at_20(percentages):
comp = ComprehensivenessMetric(
self.model,
percentages=percentages,
ablation_strategy="zero",
sample_filter=negative_filter,
)
detailed = comp.compute(
self.batch, attributions, return_per_percentage=True
)
return detailed[20]

torch.testing.assert_close(score_at_20([20]), score_at_20([10, 20]))

def test_attribution_shape_mismatch(self):
"""Test that mismatched attribution shapes are handled gracefully."""
# Skip this test - shape mismatches may not always raise errors
Expand Down