diff --git a/workspaces/scorecard/.changeset/breezy-numbers-hide.md b/workspaces/scorecard/.changeset/breezy-numbers-hide.md new file mode 100644 index 00000000000..2cb7ed9b761 --- /dev/null +++ b/workspaces/scorecard/.changeset/breezy-numbers-hide.md @@ -0,0 +1,6 @@ +--- +'@red-hat-developer-hub/backstage-plugin-scorecard-backend': minor +'@red-hat-developer-hub/backstage-plugin-scorecard-common': minor +--- + +Entity time-series API (`GET /metrics/catalog/:kind/:namespace/:name/time-series`) now returns entity-resolved `thresholds` and per-point `thresholdEvaluation` (classified at read time against those current thresholds) so clients can render sparkline legends and chart colors without a separate snapshot call. Threshold evaluation failures are returned in the existing per-point `error` field. When entity threshold resolution fails (e.g. malformed annotation overrides), the response sets `thresholdsError` and omits `thresholds` instead of silently falling back to config/provider defaults; points are left unclassified (`thresholdEvaluation` null). diff --git a/workspaces/scorecard/plugins/scorecard-backend/README.md b/workspaces/scorecard/plugins/scorecard-backend/README.md index 10d55ae4bc9..219416d4b52 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/README.md +++ b/workspaces/scorecard/plugins/scorecard-backend/README.md @@ -309,6 +309,10 @@ curl -X GET "{{url}}/api/scorecard/metrics/catalog/component/default/my-service? Returns daily time-series points for one metric on a catalog entity. Each point is the latest sample (`MAX(id)` among success or calculation-error rows) for that UTC calendar day. On a mixed day the later sample wins, so a later error is returned as `{ "value": null, "error": "..." }` (and clients can gap a sparkline). Days with no rows (or only null without `error_message`) are omitted. Returns `200` with `points: []` when the entity and metric are authorized but no data exists in the range. +The response also includes entity-resolved `thresholds` (provider defaults, then app-config, then entity annotation overrides) for sparkline legend rendering. Successful points include `thresholdEvaluation`: the matched threshold rule key from **read-time** evaluation of the point's `value` against those current `thresholds` (e.g. `success`, `warning`, `error`). This keeps legend keys and point classifications consistent when config changes. Calculation-error points omit `thresholdEvaluation`. When no rule matches, `thresholdEvaluation` is `null` (no per-point `error`). Threshold evaluation failures set that point's `error` (with `thresholdEvaluation` `null`). + +When entity threshold resolution fails (e.g. malformed annotation overrides), the response omits `thresholds`, sets top-level `thresholdsError` with the failure message, and leaves successful points unclassified (`thresholdEvaluation` `null`, no per-point `error`). There is no silent fallback to app-config / provider defaults. + #### Path Parameters | Parameter | Type | Required | Description | @@ -350,14 +354,35 @@ curl -X GET "{{url}}/api/scorecard/metrics/catalog/component/default/my-service/ "defaultVisualization": "donut" }, "points": [ - { "value": 8, "timestamp": "2026-04-27T23:10:00.000Z" }, + { + "value": 8, + "timestamp": "2026-04-27T23:10:00.000Z", + "thresholdEvaluation": "success" + }, { "value": null, "timestamp": "2026-04-28T16:00:00.000Z", "error": "GitHub API 500" }, - { "value": 7, "timestamp": "2026-04-29T22:55:00.000Z" } - ] + { + "value": 25, + "timestamp": "2026-04-29T22:55:00.000Z", + "thresholdEvaluation": "warning" + }, + { + "value": 12, + "timestamp": "2026-04-30T18:00:00.000Z", + "thresholdEvaluation": null, + "error": "Error: Invalid threshold expression" + } + ], + "thresholds": { + "rules": [ + { "key": "success", "expression": "<10" }, + { "key": "warning", "expression": "10-50" }, + { "key": "error", "expression": ">50" } + ] + } } ``` diff --git a/workspaces/scorecard/plugins/scorecard-backend/docs/thresholds.md b/workspaces/scorecard/plugins/scorecard-backend/docs/thresholds.md index d8af4cef6ee..3ba038aad2a 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/docs/thresholds.md +++ b/workspaces/scorecard/plugins/scorecard-backend/docs/thresholds.md @@ -473,7 +473,7 @@ The `ThresholdEvaluator` service processes threshold rules and determines which 1. **Order-dependent evaluation**: Rules are evaluated in the order they appear. If provider supports overriding defaults through [app configuration](#App-Configuration-Thresholds), you can change the evaluation order by specifying threshold keys in a different order. Entity annotations cannot alter the evaluation order, which is determined by either the [app configuration](#Provider-Default-Thresholds) or, if not specified, the [default provider configuration](#Provider-Default-Thresholds). 2. **First-match wins**: Returns the first threshold rule whose condition the value satisfies 3. **Type-safe**: Validates expressions against metric types -4. **Error handling**: Thresholds from providers and custom thresholds from configuration are validated on startup (using [validateThresholdsForMetric](../../scorecard-node/src/utils/thresholds/validateThresholds.ts) from `@red-hat-developer-hub/backstage-plugin-scorecard-node`). Threshold errors caused by invalid providers or invalid configuration cause startup failures. Annotation-based threshold errors are reported in the UI at evaluation time. +4. **Error handling**: Thresholds from providers and custom thresholds from configuration are validated on startup (using [validateThresholdsForMetric](../../scorecard-node/src/utils/thresholds/validateThresholds.ts) from `@red-hat-developer-hub/backstage-plugin-scorecard-node`). Threshold errors caused by invalid providers or invalid configuration cause startup failures. Annotation-based threshold errors are reported in the UI at evaluation time. On the entity time-series API (`GET /metrics/catalog/:kind/:namespace/:name/time-series`), threshold **evaluation** failures are returned in that point's `error` field (with `thresholdEvaluation` `null`). When entity threshold **resolution** fails (e.g. malformed annotation overrides), the response sets top-level `thresholdsError`, omits `thresholds` (no fallback to app-config / provider defaults), and leaves points unclassified (`thresholdEvaluation` `null`, no per-point `error`). ### Best Practices diff --git a/workspaces/scorecard/plugins/scorecard-backend/src/service/CatalogMetricService.test.ts b/workspaces/scorecard/plugins/scorecard-backend/src/service/CatalogMetricService.test.ts index 866f6ff74d2..22cd4fffbd9 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/src/service/CatalogMetricService.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend/src/service/CatalogMetricService.test.ts @@ -52,6 +52,7 @@ import { AggregatedMetricMapper } from './mappers'; import { AggregatedMetricLoader } from './aggregations/AggregatedMetricLoader'; import { DatabaseMetricValues } from '../database/DatabaseMetricValues'; import { ThresholdResolver } from '../threshold/ThresholdResolver'; +import { ThresholdEvaluator } from '../threshold/ThresholdEvaluator'; jest.mock('../permissions/permissionUtils'); @@ -532,13 +533,20 @@ describe('CatalogMetricService', () => { defaultVisualization: provider.getMetrics()[0].defaultVisualization, collectorIds: provider.getMetrics()[0].collectorIds, }, + thresholds: { rules: mockThresholdRules }, }); expect( mockedDatabase.readLatestEntityMetricValuesPerUtcDay, ).toHaveBeenCalledWith(entityRef, metricId, from, to); + expect( + mockedThresholdResolver.resolveEntityThresholds, + ).toHaveBeenCalledWith( + mockEntity, + expect.objectContaining({ id: metricId }), + ); }); - it('should map each daily DB row to a time-series point', async () => { + it('should map each daily DB row to a time-series point with read-time thresholdEvaluation', async () => { mockedDatabase.readLatestEntityMetricValuesPerUtcDay.mockResolvedValue([ { id: 3, @@ -553,9 +561,10 @@ describe('CatalogMetricService', () => { id: 2, catalogEntityRef: entityRef, metricId: metricId, - value: 7, + value: 25, timestamp: new Date('2024-01-02T12:00:00.000Z'), errorMessage: null, + // Stale write-time status must be ignored in favor of read-time evaluation status: 'success', }, ] as DbMetricValue[]); @@ -568,12 +577,21 @@ describe('CatalogMetricService', () => { ); expect(result.points).toEqual([ - { value: 9, timestamp: '2024-01-01T20:00:00.000Z' }, - { value: 7, timestamp: '2024-01-02T12:00:00.000Z' }, + { + value: 9, + timestamp: '2024-01-01T20:00:00.000Z', + thresholdEvaluation: 'success', + }, + { + value: 25, + timestamp: '2024-01-02T12:00:00.000Z', + thresholdEvaluation: 'warning', + }, ]); + expect(result.thresholds).toEqual({ rules: mockThresholdRules }); }); - it('should map calculation-error rows to null value with error', async () => { + it('should map calculation-error rows to null value with error and omit thresholdEvaluation', async () => { mockedDatabase.readLatestEntityMetricValuesPerUtcDay.mockResolvedValue([ { id: 1, @@ -612,13 +630,181 @@ describe('CatalogMetricService', () => { ); expect(result.points).toEqual([ - { value: 8, timestamp: '2024-01-01T10:00:00.000Z' }, + { + value: 8, + timestamp: '2024-01-01T10:00:00.000Z', + thresholdEvaluation: 'success', + }, + { + value: null, + timestamp: '2024-01-02T16:00:00.000Z', + error: 'GitHub API 500', + }, + { + value: 7, + timestamp: '2024-01-03T10:00:00.000Z', + thresholdEvaluation: 'success', + }, + ]); + }); + + it('should evaluate thresholdEvaluation from current thresholds even when DB status is null', async () => { + mockedDatabase.readLatestEntityMetricValuesPerUtcDay.mockResolvedValue([ + { + id: 1, + catalogEntityRef: entityRef, + metricId: metricId, + value: 5, + timestamp: new Date('2024-01-01T10:00:00.000Z'), + errorMessage: null, + status: null, + }, + ] as DbMetricValue[]); + + const result = await service.getEntityMetricTimeSeries( + entityRef, + metricId, + from, + to, + ); + + expect(result.points).toEqual([ + { + value: 5, + timestamp: '2024-01-01T10:00:00.000Z', + thresholdEvaluation: 'success', + }, + ]); + }); + + it('should set error on the point when threshold evaluation fails', async () => { + jest + .spyOn(ThresholdEvaluator.prototype, 'getFirstMatchingThreshold') + .mockImplementation(() => { + throw new Error('Invalid threshold expression'); + }); + + mockedDatabase.readLatestEntityMetricValuesPerUtcDay.mockResolvedValue([ + { + id: 1, + catalogEntityRef: entityRef, + metricId: metricId, + value: 5, + timestamp: new Date('2024-01-01T10:00:00.000Z'), + errorMessage: null, + status: null, + }, + ] as DbMetricValue[]); + + const result = await service.getEntityMetricTimeSeries( + entityRef, + metricId, + from, + to, + ); + + expect(result.points).toEqual([ + { + value: 5, + timestamp: '2024-01-01T10:00:00.000Z', + thresholdEvaluation: null, + error: 'Error: Invalid threshold expression', + }, + ]); + expect(mockedLogger.warn).toHaveBeenCalledWith( + expect.stringContaining( + `Failed to evaluate thresholds for metric '${metricId}' on entity '${entityRef}'`, + ), + ); + }); + + it('should set thresholdsError and skip classification when resolveEntityThresholds throws', async () => { + mockedThresholdResolver.resolveEntityThresholds.mockImplementation(() => { + throw new Error('Merge thresholds failed'); + }); + mockedDatabase.readLatestEntityMetricValuesPerUtcDay.mockResolvedValue([ + { + id: 1, + catalogEntityRef: entityRef, + metricId: metricId, + value: 5, + timestamp: new Date('2024-01-01T10:00:00.000Z'), + errorMessage: null, + status: null, + }, + ] as DbMetricValue[]); + + const result = await service.getEntityMetricTimeSeries( + entityRef, + metricId, + from, + to, + ); + + expect( + mockedThresholdResolver.resolveMetricThresholds, + ).not.toHaveBeenCalled(); + expect(result.thresholds).toBeUndefined(); + expect(result.thresholdsError).toBe('Error: Merge thresholds failed'); + expect(result.points).toEqual([ + { + value: 5, + timestamp: '2024-01-01T10:00:00.000Z', + thresholdEvaluation: null, + }, + ]); + expect(mockedLogger.warn).toHaveBeenCalledWith( + expect.stringContaining( + `Failed to resolve thresholds for metric '${metricId}' on entity '${entityRef}'`, + ), + ); + }); + + it('should keep calculation-error point errors when resolveEntityThresholds throws', async () => { + mockedThresholdResolver.resolveEntityThresholds.mockImplementation(() => { + throw new Error('Merge thresholds failed'); + }); + mockedDatabase.readLatestEntityMetricValuesPerUtcDay.mockResolvedValue([ + { + id: 1, + catalogEntityRef: entityRef, + metricId: metricId, + value: 5, + timestamp: new Date('2024-01-01T10:00:00.000Z'), + errorMessage: null, + status: null, + }, + { + id: 2, + catalogEntityRef: entityRef, + metricId: metricId, + value: null, + timestamp: new Date('2024-01-02T16:00:00.000Z'), + errorMessage: 'GitHub API 500', + status: null, + }, + ] as DbMetricValue[]); + + const result = await service.getEntityMetricTimeSeries( + entityRef, + metricId, + from, + to, + ); + + expect(result.thresholds).toBeUndefined(); + expect(result.thresholdsError).toBe('Error: Merge thresholds failed'); + expect(result.points).toEqual([ + { + value: 5, + timestamp: '2024-01-01T10:00:00.000Z', + thresholdEvaluation: null, + }, { value: null, timestamp: '2024-01-02T16:00:00.000Z', error: 'GitHub API 500', }, - { value: 7, timestamp: '2024-01-03T10:00:00.000Z' }, ]); }); diff --git a/workspaces/scorecard/plugins/scorecard-backend/src/service/CatalogMetricService.ts b/workspaces/scorecard/plugins/scorecard-backend/src/service/CatalogMetricService.ts index 922ad8e7743..11d6e6f2805 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/src/service/CatalogMetricService.ts +++ b/workspaces/scorecard/plugins/scorecard-backend/src/service/CatalogMetricService.ts @@ -50,6 +50,7 @@ import { isMetricCalculationError } from '../utils/metricCalculationError'; import { AggregatedMetricMapper } from './mappers'; import { DbMetricValue } from '../database/types'; import { ThresholdResolver } from '../threshold/ThresholdResolver'; +import { ThresholdEvaluator } from '../threshold/ThresholdEvaluator'; type CatalogMetricServiceOptions = { catalog: CatalogService; @@ -82,6 +83,7 @@ export class CatalogMetricService { private readonly registry: MetricProvidersRegistry; private readonly database: DatabaseMetricValues; private readonly thresholdResolver: ThresholdResolver; + private readonly thresholdEvaluator = new ThresholdEvaluator(); private static readonly MAX_FETCHABLE_ROWS = 10_000; private static readonly BATCH_SIZE = 100; @@ -194,7 +196,10 @@ export class CatalogMetricService { * * Returns at most one point per UTC calendar day: the latest sample * (`MAX(id)`), whether success or calculation error. Calculation failures - * use `value: null` and `error`. + * use `value: null` and `error`. Threshold evaluation failures also set + * `error` (with `thresholdEvaluation` null). When entity threshold + * resolution fails, `thresholdsError` is set on the response and points are + * not classified (`thresholdEvaluation` null, no per-point `error`). * * @param entityRef - Entity reference in format "kind:namespace/name" * @param metricId - Metric ID to fetch @@ -233,6 +238,20 @@ export class CatalogMetricService { to, ); + let thresholds: ThresholdConfig | undefined; + let thresholdsError: string | undefined; + try { + thresholds = this.thresholdResolver.resolveEntityThresholds( + entity, + metric, + ); + } catch (err) { + thresholdsError = stringifyError(err); + this.logger.warn( + `Failed to resolve thresholds for metric '${metric.id}' on entity '${entityRef}': ${thresholdsError}`, + ); + } + const points: MetricTimeSeriesPoint[] = rows.map(row => { if (isMetricCalculationError(row)) { return { @@ -241,9 +260,30 @@ export class CatalogMetricService { error: row.errorMessage!, }; } + + let thresholdEvaluation: string | null = null; + let error: string | undefined; + if (row.value !== null && thresholds) { + try { + thresholdEvaluation = + this.thresholdEvaluator.getFirstMatchingThreshold( + row.value, + metric.type, + thresholds, + ) ?? null; + } catch (err) { + error = stringifyError(err); + this.logger.warn( + `Failed to evaluate thresholds for metric '${metric.id}' on entity '${entityRef}': ${error}`, + ); + } + } + return { value: row.value, timestamp: row.timestamp.toISOString(), + thresholdEvaluation, + ...(error ? { error } : {}), }; }); @@ -260,6 +300,8 @@ export class CatalogMetricService { defaultVisualization: metric.defaultVisualization, collectorIds: metric.collectorIds, }, + ...(thresholds ? { thresholds } : {}), + ...(thresholdsError ? { thresholdsError } : {}), }; } diff --git a/workspaces/scorecard/plugins/scorecard-backend/src/service/router.test.ts b/workspaces/scorecard/plugins/scorecard-backend/src/service/router.test.ts index 1dfa37b2035..0f9965dfca5 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/src/service/router.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend/src/service/router.test.ts @@ -654,9 +654,24 @@ describe('createRouter', () => { defaultVisualization: 'donut', }, points: [ - { value: 8, timestamp: '2024-01-01T20:00:00.000Z' }, - { value: 7, timestamp: '2024-01-02T12:00:00.000Z' }, + { + value: 8, + timestamp: '2024-01-01T20:00:00.000Z', + thresholdEvaluation: 'success', + }, + { + value: 7, + timestamp: '2024-01-02T12:00:00.000Z', + thresholdEvaluation: 'success', + }, ], + thresholds: { + rules: [ + { key: 'error', expression: '>40' }, + { key: 'warning', expression: '>20' }, + { key: 'success', expression: '<=20' }, + ], + }, }; const timeSeriesPath = diff --git a/workspaces/scorecard/plugins/scorecard-common/report.api.md b/workspaces/scorecard/plugins/scorecard-common/report.api.md index cd0d757694e..88f6dad5b49 100644 --- a/workspaces/scorecard/plugins/scorecard-common/report.api.md +++ b/workspaces/scorecard/plugins/scorecard-common/report.api.md @@ -181,6 +181,7 @@ export type MetricTimeSeriesPoint = { value: MetricValue | null; timestamp: string; error?: string; + thresholdEvaluation?: string | null; }; // @public @@ -197,6 +198,8 @@ export type MetricTimeSeriesResponse = { defaultVisualization?: ScorecardVisualizationType; collectorIds?: string[]; }; + thresholds?: ThresholdConfig; + thresholdsError?: string; }; // @public (undocumented) diff --git a/workspaces/scorecard/plugins/scorecard-common/src/types/Metric.ts b/workspaces/scorecard/plugins/scorecard-common/src/types/Metric.ts index 2189173db92..ce4e7ca3bee 100644 --- a/workspaces/scorecard/plugins/scorecard-common/src/types/Metric.ts +++ b/workspaces/scorecard/plugins/scorecard-common/src/types/Metric.ts @@ -129,15 +129,25 @@ export type EntityMetricDetailResponse = { * A single sample in a metric time series (latest value for a UTC day). * Success points have a non-null `value`. When the latest sample is a * calculation failure, `value` is `null` and `error` is set to the failure - * message. + * message. Threshold evaluation failures also set `error` (with + * `thresholdEvaluation` null). * @public */ export type MetricTimeSeriesPoint = { value: MetricValue | null; /** ISO-8601 timestamp of the chosen sample */ timestamp: string; - /** Present when this point is a calculation failure */ + /** + * Present when this point is a calculation failure or threshold evaluation + * failed. + */ error?: string; + /** + * Matched threshold rule key from read-time evaluation against the response + * `thresholds` (e.g., "elite", "success", "warning"). + * `null` when the value could not be classified. Absent on calculation-error points. + */ + thresholdEvaluation?: string | null; }; /** @@ -157,4 +167,15 @@ export type MetricTimeSeriesResponse = { defaultVisualization?: ScorecardVisualizationType; collectorIds?: string[]; }; + /** + * Entity-resolved threshold rules (provider defaults, then app-config, then entity annotation overrides). + * Used for sparkline legend rendering and mapping `thresholdEvaluation` keys to colors. + * Undefined when entity threshold resolution failed (see `thresholdsError`). + */ + thresholds?: ThresholdConfig; + /** + * Set when entity threshold resolution failed (e.g. malformed entity annotation overrides). + * When present, points are not classified (`thresholdEvaluation` is null). + */ + thresholdsError?: string; };