[P2] UI: Drive stage Active/Running/Standby badges from live metrics

Scraping_project/dashboard/app.js

@@ -790,6 +790,26 @@ function applyStageActivity(rates) {
     });
 }
 
+// #384: Overview stage-card badges and the System Health stage rows track the
+// live sample: Running when the stage moved work in the last interval (or has
+// a queue backlog), Standby when idle with an empty queue, Unknown when the
+// stage's counter is not exported. The fetch-failure path repaints both as
+// Unknown/danger.
+const STAGE_BADGE_LABELS = { running: 'Running', standby: 'Standby', unknown: 'Unknown' };
+const STAGE_BADGE_CLASSES = { running: 'badge-success', standby: 'badge-info', unknown: 'badge-danger' };
+function applyStageBadges(metrics, rates) {
+    const states = stageBadgeStates(metrics, rates);
+    states.forEach((state, i) => {
+        const badge = document.getElementById(`overview-s${i + 1}-badge`);
+        if (badge) {
+            badge.textContent = STAGE_BADGE_LABELS[state];
+            badge.classList.remove('badge-info', 'badge-success', 'badge-danger');
+            badge.classList.add(STAGE_BADGE_CLASSES[state]);
+        }
+        setHealthTile(`stage${i + 1}-health`, `Stage ${i + 1}`, state !== 'unknown', STAGE_BADGE_LABELS[state]);
+    });
+}
+
 function calculateRates(metrics, now = Date.now()) {
     // #141: all rates are per minute, from the actual time since the last sample.
     const prev = previousMetrics;
@@ -961,6 +981,7 @@ function updateDashboard(metrics) {
     if (s4RateElem) { const __n = rates.largeDocs.toFixed(1) + '/min'; if (s4RateElem.textContent !== String(__n)) { s4RateElem.textContent = __n; s4RateElem.classList.remove('flash'); void s4RateElem.offsetWidth; s4RateElem.classList.add('flash'); } else { s4RateElem.textContent = __n; } }
 
     applyStageActivity(rates);
+    applyStageBadges(metrics, rates);
 
     setMetricText('perf-s1-rate', rates.urls.toFixed(1) + ' URLs/min');
     setMetricText('perf-s2-rate', rates.pages.toFixed(1) + ' pages/min');
@@ -1240,6 +1261,8 @@ async function fetchMetrics() {
         if (typeof setHealthTile === 'function') {
             setHealthTile('metrics-health', 'Metrics', false, 'Fetch failed');
             setHealthTile('redis-health', 'Redis', false, 'Unknown');
+            // #384: stage rows are live too; a failed fetch means Unknown.
+            for (let i = 1; i <= 4; i++) setHealthTile(`stage${i}-health`, `Stage ${i}`, false, 'Unknown');
         }
     } finally {
         if (main) main.setAttribute('aria-busy', 'false');

Scraping_project/dashboard/format-utils.js

@@ -294,6 +294,40 @@
             return Number.isFinite(v) && v > threshold;
         });
     }
+    // #384: badge state for the four Overview stage cards, derived from the
+    // live sample instead of hardcoded text. Order matches the stage cards.
+    // 'running' when the stage's per-interval rate is positive or work is
+    // queued behind it, 'standby' when the rate is ~0 with an empty queue,
+    // 'unknown' when the exporter has not reported the stage's counter series
+    // (never invent a state; the fetch-failure path repaints them as Unknown).
+    const STAGE_BADGE_DEFS = [
+        { metric: 'stage1_urls_discovered_total', rate: 'urls', counters: ['stage1_urls_queued_total'], queues: ['priority_queue'] },
+        { metric: 'stage2_pages_analyzed_total', rate: 'pages', counters: [], queues: ['stage2_queue'] },
+        { metric: 'stage3_summaries_created_total', rate: 'summaries', counters: [], queues: ['stage3_queue'] },
+        { metric: 'stage4_large_doc_summaries_total', rate: 'largeDocs', counters: [], queues: ['stage4_queue'] },
+    ];
+    function stageQueueBacklog(metrics, queueNames) {
+        let total = 0;
+        for (const key of Object.keys(metrics || {})) {
+            const m = key.match(/^redis_queue_length\{.*\bqueue="([^"]+)"/);
+            if (m && queueNames.includes(m[1])) {
+                const v = Number(metrics[key]);
+                if (Number.isFinite(v) && v > 0) total += v;
+            }
+        }
+        return total;
+    }
+    function stageBadgeStates(metrics, rates) {
+        const m = metrics || {};
+        return STAGE_BADGE_DEFS.map(def => {
+            if (!Number.isFinite(Number(m[def.metric]))) return 'unknown';
+            const rate = Number(rates && rates[def.rate]);
+            if (Number.isFinite(rate) && rate > 0) return 'running';
+            const backlog = def.counters.reduce((sum, k) => sum + (Number(m[k]) || 0), 0)
+                + stageQueueBacklog(m, def.queues);
+            return backlog > 0 ? 'running' : 'standby';
+        });
+    }
     // #412: live job progress strip. Counters come from the same sample as the
     // stage cards. A series that is not exported is null ("not reported"), never 0,
     // and no metrics at all is the empty state.
@@ -1397,5 +1431,5 @@
         return sampled;
     }
 
-    return { tabKeyTarget, staleState, queueDepthRows, metricsFailureActivity, shortcutHintText, parseMetrics, metricSeriesCount, METRICS_FLOOD_DEFAULTS, metricsFloodLimits, advanceMetricsFlood, metricsFloodBannerText, ratePerMinute, resolveMetricsUrl, metricsEndpoint, metricsEndpointLabel, formatNumber, formatBytes, formatEpochTime, countLabelValues, createRefreshScheduler, stageActivity, jobProgress, stageDrilldown, stageDrilldownKey, doughnutLegendLayout, connectionState, documentTitle, createActivityAnnouncer, metricsUrlProblem, splitLinks, createMetricsFetcher, REFRESH_INTERVAL_CHOICES, parseRefreshInterval, shortcutAction, pickInitialTab, METRIC_UI_MAP, renderMetricNameMap, parseDangerSoundEnabled, dangerSoundDecision, HEALTH_HISTORY_LIMIT, pushPollOutcome, healthHistoryView, COMPARISON_BASELINE_ROLE, PREV_HOUR_AVG_KEYS, readPrevHourAverage, comparisonBaselineSeries, comparisonNote, ACTIVITY_LOG_CAP, capActivityLog, activityStaleNote, restoreActivityLog, topScrapeTargets, SCRAPE_TARGET_EMPTY, ACTIVITY_LEVELS, activityLevelCounts, filterActivityItems, filterActivityMessages, activityFilterStatus, toggleActivityLevel, HISTORY_MAX_POINTS, HISTORY_RAW_POINTS, assertHistoryBound, boundHistoryBuffer, pushHistorySample, enforceHistoryCap, downsampleIndices, grafanaHomeUrl, jobStartedPresentation, JS_PRESET_PAGES, jsBudgetSplit, jsRenderBudget, DEFAULT_ACCENT, parseAccentColor, accentInk, accentBorder, accentCssVars };
+    return { tabKeyTarget, staleState, queueDepthRows, metricsFailureActivity, shortcutHintText, parseMetrics, metricSeriesCount, METRICS_FLOOD_DEFAULTS, metricsFloodLimits, advanceMetricsFlood, metricsFloodBannerText, ratePerMinute, resolveMetricsUrl, metricsEndpoint, metricsEndpointLabel, formatNumber, formatBytes, formatEpochTime, countLabelValues, createRefreshScheduler, stageActivity, stageBadgeStates, jobProgress, stageDrilldown, stageDrilldownKey, doughnutLegendLayout, connectionState, documentTitle, createActivityAnnouncer, metricsUrlProblem, splitLinks, createMetricsFetcher, REFRESH_INTERVAL_CHOICES, parseRefreshInterval, shortcutAction, pickInitialTab, METRIC_UI_MAP, renderMetricNameMap, parseDangerSoundEnabled, dangerSoundDecision, HEALTH_HISTORY_LIMIT, pushPollOutcome, healthHistoryView, COMPARISON_BASELINE_ROLE, PREV_HOUR_AVG_KEYS, readPrevHourAverage, comparisonBaselineSeries, comparisonNote, ACTIVITY_LOG_CAP, capActivityLog, activityStaleNote, restoreActivityLog, topScrapeTargets, SCRAPE_TARGET_EMPTY, ACTIVITY_LEVELS, activityLevelCounts, filterActivityItems, filterActivityMessages, activityFilterStatus, toggleActivityLevel, HISTORY_MAX_POINTS, HISTORY_RAW_POINTS, assertHistoryBound, boundHistoryBuffer, pushHistorySample, enforceHistoryCap, downsampleIndices, grafanaHomeUrl, jobStartedPresentation, JS_PRESET_PAGES, jsBudgetSplit, jsRenderBudget, DEFAULT_ACCENT, parseAccentColor, accentInk, accentBorder, accentCssVars };
 });

Scraping_project/dashboard/index.html

@@ -1560,7 +1560,7 @@ <h2 class="card-title" id="job-progress-title">Job progress</h2>
                     <div class="card">
                         <div class="card-header">
                             <h2 class="card-title">Stage 1: Discovery</h2>
-                            <span class="card-badge badge-info">Pending</span>
+                            <span class="card-badge badge-info" id="overview-s1-badge">Pending</span>
                         </div>
                         <div class="metric-row">
                             <span class="metric-label">URLs Found</span>
@@ -1579,7 +1579,7 @@ <h2 class="card-title">Stage 1: Discovery</h2>
                     <div class="card">
                         <div class="card-header">
                             <h2 class="card-title">Stage 2: Analysis</h2>
-                            <span class="card-badge badge-info">Pending</span>
+                            <span class="card-badge badge-info" id="overview-s2-badge">Pending</span>
                         </div>
                         <div class="metric-row">
                             <span class="metric-label">Pages Analyzed</span>
@@ -1598,7 +1598,7 @@ <h2 class="card-title">Stage 2: Analysis</h2>
                     <div class="card">
                         <div class="card-header">
                             <h2 class="card-title">Stage 3: Summarization</h2>
-                            <span class="card-badge badge-success">Running</span>
+                            <span class="card-badge badge-info" id="overview-s3-badge">Pending</span>
                         </div>
                         <div class="metric-row">
                             <span class="metric-label">Summaries</span>
@@ -1617,7 +1617,7 @@ <h2 class="card-title">Stage 3: Summarization</h2>
                     <div class="card">
                         <div class="card-header">
                             <h2 class="card-title">Stage 4: Large Docs</h2>
-                            <span class="card-badge badge-info">Standby</span>
+                            <span class="card-badge badge-info" id="overview-s4-badge">Standby</span>
                         </div>
                         <div class="metric-row">
                             <span class="metric-label">Processed</span>

Scraping_project/dashboard/tests/stage-badges.test.js

@@ -0,0 +1,70 @@
+// Run: node --test Scraping_project/dashboard/tests/
+// #384: Overview stage-card badges (and the System Health stage rows) are
+// driven by live metrics: Running when the stage's rate is positive (or work
+// is queued behind it), Standby when idle with an empty queue, Unknown when
+// the counter is not exported or the fetch failed.
+const test = require('node:test');
+const assert = require('node:assert');
+const fs = require('node:fs');
+const path = require('node:path');
+const { stageBadgeStates, parseMetrics } = require('../format-utils.js');
+
+const html = fs.readFileSync(path.join(__dirname, '..', 'index.html'), 'utf8');
+const app = fs.readFileSync(path.join(__dirname, '..', 'app.js'), 'utf8');
+
+const IDLE = [
+    'stage1_urls_discovered_total 1200',
+    'stage2_pages_analyzed_total 300',
+    'stage3_summaries_created_total 42',
+    'stage4_large_doc_summaries_total 7',
+].join('\n');
+
+test('idle metrics: every reported stage is standby', () => {
+    const rates = { urls: 0, pages: 0, summaries: 0, largeDocs: 0 };
+    assert.deepStrictEqual(stageBadgeStates(parseMetrics(IDLE), rates),
+        ['standby', 'standby', 'standby', 'standby']);
+});
+
+test('idle -> active: a positive rate flips that stage to running', () => {
+    const idle = stageBadgeStates(parseMetrics(IDLE), { urls: 0, pages: 0, summaries: 0, largeDocs: 0 });
+    const active = stageBadgeStates(parseMetrics(IDLE), { urls: 0, pages: 0, summaries: 3.5, largeDocs: 0 });
+    assert.strictEqual(idle[2], 'standby');
+    assert.strictEqual(active[2], 'running');
+    assert.deepStrictEqual(active, ['standby', 'standby', 'running', 'standby']);
+});
+
+test('zero rate with a queue backlog is running, not standby', () => {
+    const metrics = parseMetrics(IDLE + '\nredis_queue_length{queue="stage2_queue",site_id="uconn"} 5');
+    const states = stageBadgeStates(metrics, { urls: 0, pages: 0, summaries: 0, largeDocs: 0 });
+    assert.strictEqual(states[1], 'running');
+});
+
+test('stage 1 backlog comes from stage1_urls_queued_total or the priority queue', () => {
+    const rates = { urls: 0, pages: 0, summaries: 0, largeDocs: 0 };
+    assert.strictEqual(stageBadgeStates(parseMetrics(IDLE + '\nstage1_urls_queued_total 9'), rates)[0], 'running');
+    assert.strictEqual(stageBadgeStates(parseMetrics(IDLE + '\nredis_queue_length{queue="priority_queue",site_id="uconn"} 2'), rates)[0], 'running');
+});
+
+test('metrics unavailable: missing counter series is unknown, never guessed', () => {
+    assert.deepStrictEqual(stageBadgeStates(null, null),
+        ['unknown', 'unknown', 'unknown', 'unknown']);
+    assert.deepStrictEqual(stageBadgeStates(parseMetrics('stage1_urls_discovered_total 5'), { urls: 0 }),
+        ['standby', 'unknown', 'unknown', 'unknown']);
+});
+
+test('Overview stage badges carry ids and no hardcoded Running state', () => {
+    for (const id of ['overview-s1-badge', 'overview-s2-badge', 'overview-s3-badge', 'overview-s4-badge']) {
+        assert.ok(html.includes(`id="${id}"`), `missing ${id}`);
+    }
+    assert.ok(!/<span class="card-badge badge-success">Running<\/span>/.test(html));
+});
+
+test('app.js drives badges from updateDashboard and covers failure + health rows', () => {
+    assert.match(app, /applyStageBadges\(metrics, rates\)/);
+    assert.match(app, /stageBadgeStates\(metrics, rates\)/);
+    // fetch-failure path: badges become Unknown/danger, stage health rows Unknown
+    assert.match(app, /b\.textContent = 'Unknown'/);
+    assert.match(app, /setHealthTile\(`stage\$\{i\}-health`, `Stage \$\{i\}`, false, 'Unknown'\)/);
+    // health rows exist for all four stages
+    for (let i = 1; i <= 4; i++) assert.ok(html.includes(`id="stage${i}-health"`));
+});