diff --git a/Scraping_project/dashboard/app.js b/Scraping_project/dashboard/app.js index 2b87ed20..ef97ee04 100644 --- a/Scraping_project/dashboard/app.js +++ b/Scraping_project/dashboard/app.js @@ -790,6 +790,26 @@ function applyStageActivity(rates) { }); } +// #384: Overview stage-card badges and the System Health stage rows track the +// live sample: Running when the stage moved work in the last interval (or has +// a queue backlog), Standby when idle with an empty queue, Unknown when the +// stage's counter is not exported. The fetch-failure path repaints both as +// Unknown/danger. +const STAGE_BADGE_LABELS = { running: 'Running', standby: 'Standby', unknown: 'Unknown' }; +const STAGE_BADGE_CLASSES = { running: 'badge-success', standby: 'badge-info', unknown: 'badge-danger' }; +function applyStageBadges(metrics, rates) { + const states = stageBadgeStates(metrics, rates); + states.forEach((state, i) => { + const badge = document.getElementById(`overview-s${i + 1}-badge`); + if (badge) { + badge.textContent = STAGE_BADGE_LABELS[state]; + badge.classList.remove('badge-info', 'badge-success', 'badge-danger'); + badge.classList.add(STAGE_BADGE_CLASSES[state]); + } + setHealthTile(`stage${i + 1}-health`, `Stage ${i + 1}`, state !== 'unknown', STAGE_BADGE_LABELS[state]); + }); +} + function calculateRates(metrics, now = Date.now()) { // #141: all rates are per minute, from the actual time since the last sample. const prev = previousMetrics; @@ -961,6 +981,7 @@ function updateDashboard(metrics) { if (s4RateElem) { const __n = rates.largeDocs.toFixed(1) + '/min'; if (s4RateElem.textContent !== String(__n)) { s4RateElem.textContent = __n; s4RateElem.classList.remove('flash'); void s4RateElem.offsetWidth; s4RateElem.classList.add('flash'); } else { s4RateElem.textContent = __n; } } applyStageActivity(rates); + applyStageBadges(metrics, rates); setMetricText('perf-s1-rate', rates.urls.toFixed(1) + ' URLs/min'); setMetricText('perf-s2-rate', rates.pages.toFixed(1) + ' pages/min'); @@ -1240,6 +1261,8 @@ async function fetchMetrics() { if (typeof setHealthTile === 'function') { setHealthTile('metrics-health', 'Metrics', false, 'Fetch failed'); setHealthTile('redis-health', 'Redis', false, 'Unknown'); + // #384: stage rows are live too; a failed fetch means Unknown. + for (let i = 1; i <= 4; i++) setHealthTile(`stage${i}-health`, `Stage ${i}`, false, 'Unknown'); } } finally { if (main) main.setAttribute('aria-busy', 'false'); diff --git a/Scraping_project/dashboard/format-utils.js b/Scraping_project/dashboard/format-utils.js index dbf4bba7..fa093038 100644 --- a/Scraping_project/dashboard/format-utils.js +++ b/Scraping_project/dashboard/format-utils.js @@ -294,6 +294,40 @@ return Number.isFinite(v) && v > threshold; }); } + // #384: badge state for the four Overview stage cards, derived from the + // live sample instead of hardcoded text. Order matches the stage cards. + // 'running' when the stage's per-interval rate is positive or work is + // queued behind it, 'standby' when the rate is ~0 with an empty queue, + // 'unknown' when the exporter has not reported the stage's counter series + // (never invent a state; the fetch-failure path repaints them as Unknown). + const STAGE_BADGE_DEFS = [ + { metric: 'stage1_urls_discovered_total', rate: 'urls', counters: ['stage1_urls_queued_total'], queues: ['priority_queue'] }, + { metric: 'stage2_pages_analyzed_total', rate: 'pages', counters: [], queues: ['stage2_queue'] }, + { metric: 'stage3_summaries_created_total', rate: 'summaries', counters: [], queues: ['stage3_queue'] }, + { metric: 'stage4_large_doc_summaries_total', rate: 'largeDocs', counters: [], queues: ['stage4_queue'] }, + ]; + function stageQueueBacklog(metrics, queueNames) { + let total = 0; + for (const key of Object.keys(metrics || {})) { + const m = key.match(/^redis_queue_length\{.*\bqueue="([^"]+)"/); + if (m && queueNames.includes(m[1])) { + const v = Number(metrics[key]); + if (Number.isFinite(v) && v > 0) total += v; + } + } + return total; + } + function stageBadgeStates(metrics, rates) { + const m = metrics || {}; + return STAGE_BADGE_DEFS.map(def => { + if (!Number.isFinite(Number(m[def.metric]))) return 'unknown'; + const rate = Number(rates && rates[def.rate]); + if (Number.isFinite(rate) && rate > 0) return 'running'; + const backlog = def.counters.reduce((sum, k) => sum + (Number(m[k]) || 0), 0) + + stageQueueBacklog(m, def.queues); + return backlog > 0 ? 'running' : 'standby'; + }); + } // #412: live job progress strip. Counters come from the same sample as the // stage cards. A series that is not exported is null ("not reported"), never 0, // and no metrics at all is the empty state. @@ -1397,5 +1431,5 @@ return sampled; } - return { tabKeyTarget, staleState, queueDepthRows, metricsFailureActivity, shortcutHintText, parseMetrics, metricSeriesCount, METRICS_FLOOD_DEFAULTS, metricsFloodLimits, advanceMetricsFlood, metricsFloodBannerText, ratePerMinute, resolveMetricsUrl, metricsEndpoint, metricsEndpointLabel, formatNumber, formatBytes, formatEpochTime, countLabelValues, createRefreshScheduler, stageActivity, jobProgress, stageDrilldown, stageDrilldownKey, doughnutLegendLayout, connectionState, documentTitle, createActivityAnnouncer, metricsUrlProblem, splitLinks, createMetricsFetcher, REFRESH_INTERVAL_CHOICES, parseRefreshInterval, shortcutAction, pickInitialTab, METRIC_UI_MAP, renderMetricNameMap, parseDangerSoundEnabled, dangerSoundDecision, HEALTH_HISTORY_LIMIT, pushPollOutcome, healthHistoryView, COMPARISON_BASELINE_ROLE, PREV_HOUR_AVG_KEYS, readPrevHourAverage, comparisonBaselineSeries, comparisonNote, ACTIVITY_LOG_CAP, capActivityLog, activityStaleNote, restoreActivityLog, topScrapeTargets, SCRAPE_TARGET_EMPTY, ACTIVITY_LEVELS, activityLevelCounts, filterActivityItems, filterActivityMessages, activityFilterStatus, toggleActivityLevel, HISTORY_MAX_POINTS, HISTORY_RAW_POINTS, assertHistoryBound, boundHistoryBuffer, pushHistorySample, enforceHistoryCap, downsampleIndices, grafanaHomeUrl, jobStartedPresentation, JS_PRESET_PAGES, jsBudgetSplit, jsRenderBudget, DEFAULT_ACCENT, parseAccentColor, accentInk, accentBorder, accentCssVars }; + return { tabKeyTarget, staleState, queueDepthRows, metricsFailureActivity, shortcutHintText, parseMetrics, metricSeriesCount, METRICS_FLOOD_DEFAULTS, metricsFloodLimits, advanceMetricsFlood, metricsFloodBannerText, ratePerMinute, resolveMetricsUrl, metricsEndpoint, metricsEndpointLabel, formatNumber, formatBytes, formatEpochTime, countLabelValues, createRefreshScheduler, stageActivity, stageBadgeStates, jobProgress, stageDrilldown, stageDrilldownKey, doughnutLegendLayout, connectionState, documentTitle, createActivityAnnouncer, metricsUrlProblem, splitLinks, createMetricsFetcher, REFRESH_INTERVAL_CHOICES, parseRefreshInterval, shortcutAction, pickInitialTab, METRIC_UI_MAP, renderMetricNameMap, parseDangerSoundEnabled, dangerSoundDecision, HEALTH_HISTORY_LIMIT, pushPollOutcome, healthHistoryView, COMPARISON_BASELINE_ROLE, PREV_HOUR_AVG_KEYS, readPrevHourAverage, comparisonBaselineSeries, comparisonNote, ACTIVITY_LOG_CAP, capActivityLog, activityStaleNote, restoreActivityLog, topScrapeTargets, SCRAPE_TARGET_EMPTY, ACTIVITY_LEVELS, activityLevelCounts, filterActivityItems, filterActivityMessages, activityFilterStatus, toggleActivityLevel, HISTORY_MAX_POINTS, HISTORY_RAW_POINTS, assertHistoryBound, boundHistoryBuffer, pushHistorySample, enforceHistoryCap, downsampleIndices, grafanaHomeUrl, jobStartedPresentation, JS_PRESET_PAGES, jsBudgetSplit, jsRenderBudget, DEFAULT_ACCENT, parseAccentColor, accentInk, accentBorder, accentCssVars }; }); diff --git a/Scraping_project/dashboard/index.html b/Scraping_project/dashboard/index.html index 83fe6353..4ce26244 100644 --- a/Scraping_project/dashboard/index.html +++ b/Scraping_project/dashboard/index.html @@ -1560,7 +1560,7 @@

Job progress

Stage 1: Discovery

- Pending + Pending
URLs Found @@ -1579,7 +1579,7 @@

Stage 1: Discovery

Stage 2: Analysis

- Pending + Pending
Pages Analyzed @@ -1598,7 +1598,7 @@

Stage 2: Analysis

Stage 3: Summarization

- Running + Pending
Summaries @@ -1617,7 +1617,7 @@

Stage 3: Summarization

Stage 4: Large Docs

- Standby + Standby
Processed diff --git a/Scraping_project/dashboard/tests/stage-badges.test.js b/Scraping_project/dashboard/tests/stage-badges.test.js new file mode 100644 index 00000000..ec8e486e --- /dev/null +++ b/Scraping_project/dashboard/tests/stage-badges.test.js @@ -0,0 +1,70 @@ +// Run: node --test Scraping_project/dashboard/tests/ +// #384: Overview stage-card badges (and the System Health stage rows) are +// driven by live metrics: Running when the stage's rate is positive (or work +// is queued behind it), Standby when idle with an empty queue, Unknown when +// the counter is not exported or the fetch failed. +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const path = require('node:path'); +const { stageBadgeStates, parseMetrics } = require('../format-utils.js'); + +const html = fs.readFileSync(path.join(__dirname, '..', 'index.html'), 'utf8'); +const app = fs.readFileSync(path.join(__dirname, '..', 'app.js'), 'utf8'); + +const IDLE = [ + 'stage1_urls_discovered_total 1200', + 'stage2_pages_analyzed_total 300', + 'stage3_summaries_created_total 42', + 'stage4_large_doc_summaries_total 7', +].join('\n'); + +test('idle metrics: every reported stage is standby', () => { + const rates = { urls: 0, pages: 0, summaries: 0, largeDocs: 0 }; + assert.deepStrictEqual(stageBadgeStates(parseMetrics(IDLE), rates), + ['standby', 'standby', 'standby', 'standby']); +}); + +test('idle -> active: a positive rate flips that stage to running', () => { + const idle = stageBadgeStates(parseMetrics(IDLE), { urls: 0, pages: 0, summaries: 0, largeDocs: 0 }); + const active = stageBadgeStates(parseMetrics(IDLE), { urls: 0, pages: 0, summaries: 3.5, largeDocs: 0 }); + assert.strictEqual(idle[2], 'standby'); + assert.strictEqual(active[2], 'running'); + assert.deepStrictEqual(active, ['standby', 'standby', 'running', 'standby']); +}); + +test('zero rate with a queue backlog is running, not standby', () => { + const metrics = parseMetrics(IDLE + '\nredis_queue_length{queue="stage2_queue",site_id="uconn"} 5'); + const states = stageBadgeStates(metrics, { urls: 0, pages: 0, summaries: 0, largeDocs: 0 }); + assert.strictEqual(states[1], 'running'); +}); + +test('stage 1 backlog comes from stage1_urls_queued_total or the priority queue', () => { + const rates = { urls: 0, pages: 0, summaries: 0, largeDocs: 0 }; + assert.strictEqual(stageBadgeStates(parseMetrics(IDLE + '\nstage1_urls_queued_total 9'), rates)[0], 'running'); + assert.strictEqual(stageBadgeStates(parseMetrics(IDLE + '\nredis_queue_length{queue="priority_queue",site_id="uconn"} 2'), rates)[0], 'running'); +}); + +test('metrics unavailable: missing counter series is unknown, never guessed', () => { + assert.deepStrictEqual(stageBadgeStates(null, null), + ['unknown', 'unknown', 'unknown', 'unknown']); + assert.deepStrictEqual(stageBadgeStates(parseMetrics('stage1_urls_discovered_total 5'), { urls: 0 }), + ['standby', 'unknown', 'unknown', 'unknown']); +}); + +test('Overview stage badges carry ids and no hardcoded Running state', () => { + for (const id of ['overview-s1-badge', 'overview-s2-badge', 'overview-s3-badge', 'overview-s4-badge']) { + assert.ok(html.includes(`id="${id}"`), `missing ${id}`); + } + assert.ok(!/Running<\/span>/.test(html)); +}); + +test('app.js drives badges from updateDashboard and covers failure + health rows', () => { + assert.match(app, /applyStageBadges\(metrics, rates\)/); + assert.match(app, /stageBadgeStates\(metrics, rates\)/); + // fetch-failure path: badges become Unknown/danger, stage health rows Unknown + assert.match(app, /b\.textContent = 'Unknown'/); + assert.match(app, /setHealthTile\(`stage\$\{i\}-health`, `Stage \$\{i\}`, false, 'Unknown'\)/); + // health rows exist for all four stages + for (let i = 1; i <= 4; i++) assert.ok(html.includes(`id="stage${i}-health"`)); +});