fix(runner): cascade-skip downstream stateful steps when phase capability gate unmet

.changeset/calm-runners-skip-capabilities.md

@@ -0,0 +1,5 @@
+---
+'@adcp/sdk': patch
+---
+
+Prevent capability-gated storyboard phases from dispatching dependent requests with unavailable context and keep those applicability skips neutral in compliance bundle scoring.

src/lib/testing/compliance/comply.ts

@@ -776,7 +776,14 @@ export interface ComplianceBundleAssessmentOptions {
   failingBundleIds?: readonly string[];
 }
 
-const NEUTRAL_BUNDLE_SKIP_REASONS = new Set<string>(['peer_branch_taken', 'peer_substituted']);
+const NEUTRAL_BUNDLE_SKIP_REASONS = new Set<string>([
+  'peer_branch_taken',
+  'peer_substituted',
+  // The runner emits this only when an explicit phase capability gate made
+  // the prerequisite state unavailable. Ordinary prerequisite_failed skips
+  // remain coverage gaps and therefore keep the bundle partial.
+  'capability_prerequisite_unavailable',
+]);
 
 /**
  * Aggregate the exact cache bundles selected for a capability-driven run.
@@ -809,15 +816,42 @@ export function buildComplianceBundleResults(
       const branchSetPhaseIds = new Set(
         storyboard.phases.filter(phase => phase.branch_set !== undefined).map(phase => phase.id)
       );
+      const phaseDefs = new Map(storyboard.phases.map(phase => [phase.id, phase]));
       const hasCoverageGapSkip = (result.passes?.flatMap(pass => pass.phases) ?? result.phases).some(phase =>
         phase.steps.some(step => {
           if (!step.skipped && step.skip === undefined && step.skip_reason === undefined) return false;
-          const reason = step.skip?.reason ?? step.skip_reason;
-          if (reason !== undefined && NEUTRAL_BUNDLE_SKIP_REASONS.has(reason)) return false;
+          // A phase-level capability gate deliberately emits the protocol's
+          // canonical not_applicable reason for every step. It is complete
+          // applicability evidence, not a generic coverage gap. Keep this
+          // phase-scoped so an unrelated not_applicable skip remains partial.
+          const phaseDef = phaseDefs.get(phase.phase_id);
+          if (
+            result.overall_passed &&
+            phaseDef?.requires_capability !== undefined &&
+            phase.steps.length > 0 &&
+            phase.steps.every(
+              candidate =>
+                candidate.skipped === true && (candidate.skip?.reason ?? candidate.skip_reason) === 'not_applicable'
+            )
+          ) {
+            return false;
+          }
+          // The output-contract `skip.reason` intentionally canonicalizes
+          // detailed runner reasons. Prefer the detailed field here so a
+          // capability-gated prerequisite can be neutral without making all
+          // canonical not_applicable skips neutral coverage.
+          const detailedReason = step.skip_reason;
+          const canonicalReason = step.skip?.reason ?? detailedReason;
+          if (
+            (detailedReason !== undefined && NEUTRAL_BUNDLE_SKIP_REASONS.has(detailedReason)) ||
+            (canonicalReason !== undefined && NEUTRAL_BUNDLE_SKIP_REASONS.has(canonicalReason))
+          ) {
+            return false;
+          }
           // A successful authored any-of branch makes its unselected peers
           // legitimately not applicable; generic not_applicable skips remain
           // coverage gaps everywhere else.
-          if (reason === 'not_applicable' && result.overall_passed && branchSetPhaseIds.has(phase.phase_id)) {
+          if (canonicalReason === 'not_applicable' && result.overall_passed && branchSetPhaseIds.has(phase.phase_id)) {
             return false;
           }
           return true;

src/lib/testing/storyboard/runner.ts

@@ -2651,6 +2651,10 @@ async function executeStoryboardPass(
   const priorA2aEnvelopes = new Map<string, A2ATaskEnvelope>();
   const stepRequestStarts = new Map<string, string>();
   const responseDerivedNotApplicableContextKeys = new Map<string, string>();
+  // Context outputs by capability-gated phase. The per-step execution state
+  // receives only keys from phases it actually depends on, preserving the
+  // declared depends_on / any_of branch-set topology.
+  const capabilityUnavailableContextKeysByPhase = new Map<string, Set<string>>();
   const phaseResults: StoryboardPhaseResult[] = [];
   let passedCount = 0;
   let failedCount = 0;
@@ -2684,6 +2688,8 @@ async function executeStoryboardPass(
     stepId: string;
     reason: RunnerSkipReason | RunnerDetailedSkipReason;
     substitution_chain?: string;
+    /** The trigger is an explicit phase capability gate, not a failed setup. */
+    capabilityUnavailable?: boolean;
   };
   const phaseStatefulCascades = new Map<string, CascadeTrigger | null>();
   // Phase IDs in declaration order, accumulated as we iterate so the
@@ -2734,6 +2740,19 @@ async function executeStoryboardPass(
     }
     return { tripped: false };
   };
+  const capabilityUnavailableContextKeysForPhase = (
+    phase: { id: string; depends_on?: string[] },
+    prior: readonly string[]
+  ): Set<string> => {
+    const keys = new Set<string>();
+    const ownSpec = branchSetsByPhaseId.get(phase.id);
+    const ownAnyOfBranchSet = ownSpec?.semantics === 'any_of' ? ownSpec.id : undefined;
+    for (const depId of effectiveDependsOn(phase, prior)) {
+      if (ownAnyOfBranchSet !== undefined && branchSetsByPhaseId.get(depId)?.id === ownAnyOfBranchSet) continue;
+      for (const key of capabilityUnavailableContextKeysByPhase.get(depId) ?? []) keys.add(key);
+    }
+    return keys;
+  };
   // Step results whose failures the main loop added to failedCount. The
   // branch-set post-pass decrements only for entries that were actually
   // counted, so an optional phase that hit `presenceDetected` (a PRM 2xx
@@ -3096,6 +3115,26 @@ async function executeStoryboardPass(
       });
       skippedCount += skippedSteps.length;
       phaseCapabilitySkippedIds.add(phase.id);
+      // A capability-gated phase can make downstream state unavailable either
+      // through an explicitly stateful setup step or an author-declared
+      // context output on an otherwise stateless step. Preserve the existing
+      // depends_on / any_of branch-set cascade semantics for stateful
+      // consumers, while executeStep handles non-stateful consumers that
+      // directly reference one of these keys.
+      const unavailableKeys = new Set<string>();
+      for (const step of phase.steps) {
+        for (const output of step.context_outputs ?? []) {
+          if (output.key) unavailableKeys.add(output.key);
+        }
+      }
+      capabilityUnavailableContextKeysByPhase.set(phase.id, unavailableKeys);
+      if (phase.steps.some(s => s.stateful)) {
+        phaseStatefulCascades.set(phase.id, {
+          stepId: phase.steps.find(s => s.stateful)!.id,
+          reason: 'not_applicable',
+          capabilityUnavailable: true,
+        });
+      }
       priorPhaseIds.push(phase.id);
       continue;
     }
@@ -3384,24 +3423,25 @@ async function executeStoryboardPass(
             ? `Skipped: prior stateful step "${trigger.stepId}" skipped (${trigger.reason}); ${trigger.substitution_chain}; state never materialized.`
             : `Skipped: prior stateful step "${trigger.stepId}" skipped (${trigger.reason}); state never materialized.`
           : 'Skipped: prior stateful step failed.';
+        const capabilityUnavailable = trigger?.capabilityUnavailable === true;
         stepResults.push({
           storyboard_id: storyboard.id,
           step_id: step.id,
           phase_id: phase.id,
           title: step.title,
           task: step.task,
-          passed: false,
+          passed: capabilityUnavailable,
           skipped: true,
-          skip_reason: 'prerequisite_failed',
-          skip: buildSkip('prerequisite_failed', detail),
+          skip_reason: capabilityUnavailable ? 'capability_prerequisite_unavailable' : 'prerequisite_failed',
+          skip: buildSkip(capabilityUnavailable ? 'not_applicable' : 'prerequisite_failed', detail),
           duration_ms: 0,
           validations: [],
           context,
-          error: detail,
+          ...(!capabilityUnavailable && { error: detail }),
           extraction: { path: 'none' },
         });
         skippedCount++;
-        phasePassed = false;
+        if (!capabilityUnavailable) phasePassed = false;
         continue;
       }
 
@@ -3437,6 +3477,10 @@ async function executeStoryboardPass(
         continue;
       }
       const stepExecutionState = buildExecutionState(assignment.agentUrl, assignment.profile);
+      stepExecutionState.capabilityUnavailableContextKeys = capabilityUnavailableContextKeysForPhase(
+        phase,
+        priorPhaseIds
+      );
       const rawResult = await executeStep(
         assignment.client,
         step,
@@ -4571,6 +4615,8 @@ interface ExecutionState {
    * rather than prerequisite_failed.
    */
   responseDerivedNotApplicableContextKeys?: Map<string, string>;
+  /** Context keys from capability-gated phases this step depends on. */
+  capabilityUnavailableContextKeys?: Set<string>;
   /** Shared ephemeral webhook receiver, when the run has one enabled. */
   webhookReceiver?: WebhookReceiver;
   /** Shared runner-variable bag for `{{runner.*}}` substitution. */
@@ -4638,6 +4684,7 @@ async function executeStep(
     contextProvenance: new Map(),
     stepRequestStarts: new Map(),
     responseDerivedNotApplicableContextKeys: new Map(),
+    capabilityUnavailableContextKeys: new Set(),
   };
 
   // Recognize the dedicated TMP publisher-auth probes before generic auth
@@ -4844,6 +4891,35 @@ async function executeStep(
     request = applyContextInputs(request, step.context_inputs, context);
   }
 
+  // applyContextInputs intentionally leaves absent keys alone. When such a
+  // key belongs to an unavailable capability-gated dependency, stop before
+  // dispatch rather than letting an expect_error vector accidentally test the
+  // runner's missing state.
+  const unavailableContextInputs = (step.context_inputs ?? []).filter(
+    input => !(input.key in context) && runState.capabilityUnavailableContextKeys?.has(input.key) === true
+  );
+  if (unavailableContextInputs.length > 0) {
+    const detail =
+      'Skipped: context required by a capability-gated phase is unavailable: ' +
+      unavailableContextInputs.map(input => input.key).join(', ') +
+      '.';
+    return {
+      step_id: step.id,
+      phase_id: phaseId,
+      title: step.title,
+      task: step.task,
+      passed: true,
+      skipped: true,
+      skip_reason: 'capability_prerequisite_unavailable',
+      skip: buildSkip('not_applicable', detail),
+      duration_ms: 0,
+      validations: [],
+      context,
+      next: getNextStepPreview(step.id, allSteps, context, runState.runnerVars),
+      extraction: { path: 'none' },
+    };
+  }
+
   // Brand/account is a storyboard-run-scoped invariant: every step in a run
   // targets the same brand, so every outgoing request's brand context must
   // match the options. Enforcing this here (after builder + sample_request)
@@ -4932,24 +5008,38 @@ async function executeStep(
     token: BUILD_ASSETS_FROM_FORMAT_DIRECTIVE,
   }));
   const unresolvedVars = [...unresolvedContextVars, ...unresolvedAssetDirectives];
-  // expect_error steps may intentionally preserve invalid $context tokens, but
-  // runner-only creative directives must never cross the wire.
-  if (unresolvedAssetDirectives.length > 0 || (unresolvedContextVars.length > 0 && !step.expect_error)) {
+  // Keep expect_error's intentional malformed-vector behavior, except when
+  // the unresolved token belongs to a capability-gated phase this step
+  // depends on. That token is runner state that cannot materialize and must
+  // never cross the wire.
+  const hasCapabilityUnavailableContext = unresolvedContextVars.some(
+    v => runState.capabilityUnavailableContextKeys?.has(v.key) === true
+  );
+  if (
+    unresolvedAssetDirectives.length > 0 ||
+    (unresolvedContextVars.length > 0 && (!step.expect_error || hasCapabilityUnavailableContext))
+  ) {
     const next = getNextStepPreview(step.id, allSteps, context, runState.runnerVars);
     const responseDerivedDetails = unresolvedVars
       .map(v => runState.responseDerivedNotApplicableContextKeys?.get(v.key))
       .filter((d): d is string => typeof d === 'string');
     const allResponseDerived =
       responseDerivedDetails.length === unresolvedVars.length && responseDerivedDetails.length > 0;
+    const allCapabilityUnavailable =
+      unresolvedContextVars.length === unresolvedVars.length &&
+      unresolvedContextVars.length > 0 &&
+      unresolvedContextVars.every(v => runState.capabilityUnavailableContextKeys?.has(v.key) === true);
     const detail = allResponseDerived
       ? [...new Set(responseDerivedDetails)].join('; ')
-      : `Skipped: unresolved context variables from prior steps: ${unresolvedVars.map(v => v.key).join(', ')}.`;
+      : allCapabilityUnavailable
+        ? `Skipped: context required by a capability-gated phase is unavailable: ${unresolvedVars.map(v => v.key).join(', ')}.`
+        : `Skipped: unresolved context variables from prior steps: ${unresolvedVars.map(v => v.key).join(', ')}.`;
     // Normal unresolved substitutions carry one validation result per missing
     // token. Response-derived terminal-page skips are already successful
     // not_applicable rows, so their downstream cursor consumers stay validation
     // empty to avoid inventing a failing-looking check for an expected skip.
     const synthesized: ValidationResult[] = [];
-    if (!allResponseDerived) {
+    if (!allResponseDerived && !allCapabilityUnavailable) {
       const seenTokens = new Set<string>();
       for (const v of unresolvedVars) {
         if (seenTokens.has(v.token)) continue;
@@ -4974,15 +5064,22 @@ async function executeStep(
       phase_id: phaseId,
       title: step.title,
       task: step.task,
-      passed: allResponseDerived,
+      passed: allResponseDerived || allCapabilityUnavailable,
       skipped: true,
-      skip_reason: allResponseDerived ? 'not_applicable' : 'prerequisite_failed',
-      skip: buildSkip(allResponseDerived ? 'not_applicable' : 'prerequisite_failed', detail),
+      skip_reason: allCapabilityUnavailable
+        ? 'capability_prerequisite_unavailable'
+        : allResponseDerived
+          ? 'not_applicable'
+          : 'prerequisite_failed',
+      skip: buildSkip(
+        allResponseDerived || allCapabilityUnavailable ? 'not_applicable' : 'prerequisite_failed',
+        detail
+      ),
       duration_ms: 0,
       validations: synthesized,
       context,
       ...responseDerivedContextResult(runState),
-      ...(!allResponseDerived && { error: detail }),
+      ...(!allResponseDerived && !allCapabilityUnavailable && { error: detail }),
       next,
       extraction: { path: 'none' },
     };

src/lib/testing/storyboard/types.ts

@@ -2136,6 +2136,13 @@ export type RunnerDetailedSkipReason =
    * self-declared capability profile.
    */
   | 'capability_unsupported'
+  /**
+   * A preceding phase was skipped because the agent explicitly does not
+   * support its required capability, so this step's prerequisite state cannot
+   * exist. This is a neutral applicability outcome, unlike an ordinary
+   * `prerequisite_failed` skip which remains actionable.
+   */
+  | 'capability_prerequisite_unavailable'
   /**
    * A `comply_test_controller` step targeted a `force_*` scenario that the
    * agent advertised the controller for but did not implement. Detected by
@@ -2167,6 +2174,7 @@ export const DETAILED_SKIP_TO_CANONICAL: Record<RunnerDetailedSkipReason, Runner
   fixture_seed_unsupported: 'not_applicable',
   fixture_unsatisfied: 'not_applicable',
   capability_unsupported: 'not_applicable',
+  capability_prerequisite_unavailable: 'not_applicable',
   rate_abuse_opt_out: 'unsatisfied_contract',
   missing_test_kit_contract: 'unsatisfied_contract',
   live_side_effect_opt_in_required: 'unsatisfied_contract',