diff --git a/.changeset/multi-source-select-builder.md b/.changeset/multi-source-select-builder.md new file mode 100644 index 0000000000..acfa7119e0 --- /dev/null +++ b/.changeset/multi-source-select-builder.md @@ -0,0 +1,9 @@ +--- +'@hyperdx/common-utils': minor +--- + +Add a builder for search queries that project a canonical, source-independent +column set. Given a source, it emits that source's semantic expressions +(timestamp, service, severity/status, body/span name, duration) under shared +aliases, and pads columns a source doesn't have with NULL, so results from +different tables share one shape. diff --git a/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts b/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts index 99d28a5c53..aff89d08ce 100644 --- a/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts +++ b/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts @@ -1,6 +1,9 @@ import { ALERT_COUNT_DEFAULT_SELECT, + buildMultiSourceSearchConfig, + buildMultiSourceSelect, buildSearchChartConfig, + MULTI_SOURCE_ALIASES, } from '@/core/searchChartConfig'; import { DisplayType, Filter, SourceKind, TSource } from '@/types'; @@ -477,3 +480,120 @@ describe('buildSearchChartConfig', () => { }); }); }); + +describe('buildMultiSourceSelect', () => { + it('projects every canonical alias, so callers can rely on the constants', () => { + const select = buildMultiSourceSelect(makeTraceSource(), { + includeDuration: true, + }); + + for (const alias of Object.values(MULTI_SOURCE_ALIASES)) { + expect(select).toContain(`AS "${alias}"`); + } + }); + + it('projects the canonical aliases from a Log source semantic expressions', () => { + const source = makeLogSource({ + displayedTimestampValueExpression: 'Timestamp', + serviceNameExpression: 'ServiceName', + severityTextExpression: 'SeverityText', + bodyExpression: 'Body', + }); + + expect(buildMultiSourceSelect(source)).toBe( + 'Timestamp AS "__hdx_timestamp", ' + + 'ServiceName AS "__hdx_service", ' + + 'SeverityText AS "__hdx_severity", ' + + 'Body AS "__hdx_body"', + ); + }); + + it('falls back to the first timestamp expression and NULL for missing semantics', () => { + const source = makeLogSource({ + timestampValueExpression: 'TimestampTime, Timestamp', + implicitColumnExpression: undefined, + }); + + expect(buildMultiSourceSelect(source)).toBe( + 'TimestampTime AS "__hdx_timestamp", ' + + 'NULL AS "__hdx_service", ' + + 'NULL AS "__hdx_severity", ' + + 'NULL AS "__hdx_body"', + ); + }); + + it('maps Trace sources onto status/span-name and a milliseconds duration', () => { + const source = makeTraceSource({ + serviceNameExpression: 'ServiceName', + statusCodeExpression: 'StatusCode', + spanNameExpression: 'SpanName', + durationExpression: 'Duration', + durationPrecision: 9, + }); + + expect(buildMultiSourceSelect(source, { includeDuration: true })).toBe( + 'Timestamp AS "__hdx_timestamp", ' + + 'ServiceName AS "__hdx_service", ' + + 'StatusCode AS "__hdx_severity", ' + + 'SpanName AS "__hdx_body", ' + + '(Duration)/1e6 AS "__hdx_duration_ms"', + ); + }); + + it('projects NULL duration for Log sources when duration is included', () => { + const source = makeLogSource({ bodyExpression: 'Body' }); + + expect(buildMultiSourceSelect(source, { includeDuration: true })).toContain( + 'NULL AS "__hdx_duration_ms"', + ); + }); + + it('appends extra columns, projecting NULL where a source lacks the column', () => { + const source = makeLogSource({ bodyExpression: 'Body' }); + + const select = buildMultiSourceSelect(source, { + extraColumns: [ + { name: 'ServiceName', expression: 'ServiceName' }, + { name: 'StatusCode', expression: null }, + ], + }); + + expect(select).toContain('ServiceName AS "ServiceName"'); + expect(select).toContain('NULL AS "StatusCode"'); + }); +}); + +describe('buildMultiSourceSearchConfig', () => { + it('keeps the standard search config assembly but swaps in the canonical SELECT', () => { + const source = makeLogSource({ + bodyExpression: 'Body', + tableFilterExpression: "ServiceName != 'noisy'", + }); + + const config = buildMultiSourceSearchConfig(source, { + where: 'error', + whereLanguage: 'lucene', + orderBy: 'TimestampTime DESC', + }); + + expect(config.select).toBe(buildMultiSourceSelect(source)); + expect(config.from).toEqual(source.from); + expect(config.connection).toBe('conn-1'); + expect(config.where).toBe('error'); + expect(config.whereLanguage).toBe('lucene'); + expect(config.orderBy).toBe('TimestampTime DESC'); + // Source-level behaviors (e.g. tableFilterExpression) still apply. + expect(config.filters).toEqual([ + { type: 'sql', condition: "ServiceName != 'noisy'" }, + ]); + }); + + it('never resolves to defaultTableSelectExpression', () => { + const config = buildMultiSourceSearchConfig(makeTraceSource(), { + where: '', + }); + + expect(config.select).not.toContain('SpanName,'); + expect(config.select).toContain('AS "__hdx_timestamp"'); + }); +}); diff --git a/packages/common-utils/src/core/searchChartConfig.ts b/packages/common-utils/src/core/searchChartConfig.ts index 37144db8ab..04d7dcfdca 100644 --- a/packages/common-utils/src/core/searchChartConfig.ts +++ b/packages/common-utils/src/core/searchChartConfig.ts @@ -1,3 +1,4 @@ +import { getFirstTimestampValueExpression } from '@/core/utils'; import { BuilderChartConfig, DateRange, @@ -185,3 +186,143 @@ export function buildSearchChartConfig( return config; } + +/** + * Canonical result-column aliases used when searching across multiple sources + * at once. Every selected source's SELECT is rewritten to this shape, so the + * merged results table can map columns by name regardless of how each source's + * underlying schema names them. + * + * The names are quoted aliases (`expr AS "__hdx_timestamp"`), so ClickHouse + * returns them verbatim — unlike raw expressions, which CH may reformat. + */ +export const MULTI_SOURCE_ALIASES = { + timestamp: '__hdx_timestamp', + service: '__hdx_service', + severity: '__hdx_severity', + body: '__hdx_body', + /** Milliseconds; only projected when a Trace source is in the selection. */ + durationMs: '__hdx_duration_ms', +} as const; + +/** + * An extra user-picked column to project alongside the canonical aliases. + * `expression` is the per-source SQL expression for the column, or null when + * the source has no such column (projected as NULL so every source returns + * the same column set). + */ +export type MultiSourceExtraColumn = { + /** Result column name (used verbatim as the quoted alias). */ + name: string; + expression: string | null; +}; + +const quoteAlias = (name: string) => `"${name.replace(/"/g, '\\"')}"`; + +/** + * Per-source semantic expression for each canonical alias. Mirrors the app's + * display helpers (`getDisplayedTimestampValueExpression`, `getEventBody`, + * `getDurationMsExpression` in packages/app/src/source.ts) — keep in sync. + */ +function multiSourceSemanticExpressions(source: TSource): { + timestamp: string; + service: string; + severity: string; + body: string; + durationMs: string; +} { + const firstTimestamp = getFirstTimestampValueExpression( + source.timestampValueExpression, + ); + + if (isLogSource(source)) { + return { + timestamp: source.displayedTimestampValueExpression || firstTimestamp, + service: source.serviceNameExpression || 'NULL', + severity: source.severityTextExpression || 'NULL', + body: source.bodyExpression || source.implicitColumnExpression || 'NULL', + durationMs: 'NULL', + }; + } + + if (isTraceSource(source)) { + return { + timestamp: source.displayedTimestampValueExpression || firstTimestamp, + service: source.serviceNameExpression || 'NULL', + severity: source.statusCodeExpression || 'NULL', + body: source.spanNameExpression || 'NULL', + // Match getDurationMsExpression: durationPrecision is the sub-second + // digit count (9 = nanoseconds), so /1e(precision-3) yields milliseconds. + durationMs: `(${source.durationExpression})/1e${(source.durationPrecision ?? 9) - 3}`, + }; + } + + // Multi-source search only supports Log and Trace sources today; other kinds + // still get a valid (if minimal) shape so a stray source can't render SQL + // that errors the whole selection. + return { + timestamp: firstTimestamp, + service: 'NULL', + severity: 'NULL', + body: 'NULL', + durationMs: 'NULL', + }; +} + +/** + * Build the canonical aliased SELECT string for one source in a multi-source + * search. Exported for tests. + */ +export function buildMultiSourceSelect( + source: TSource, + { + includeDuration = false, + extraColumns = [], + }: { + /** Project `__hdx_duration_ms` (set when any selected source is a Trace). */ + includeDuration?: boolean; + extraColumns?: MultiSourceExtraColumn[]; + } = {}, +): string { + const exprs = multiSourceSemanticExpressions(source); + + const parts = [ + `${exprs.timestamp} AS ${quoteAlias(MULTI_SOURCE_ALIASES.timestamp)}`, + `${exprs.service} AS ${quoteAlias(MULTI_SOURCE_ALIASES.service)}`, + `${exprs.severity} AS ${quoteAlias(MULTI_SOURCE_ALIASES.severity)}`, + `${exprs.body} AS ${quoteAlias(MULTI_SOURCE_ALIASES.body)}`, + ]; + if (includeDuration) { + parts.push( + `${exprs.durationMs} AS ${quoteAlias(MULTI_SOURCE_ALIASES.durationMs)}`, + ); + } + for (const col of extraColumns) { + parts.push(`${col.expression ?? 'NULL'} AS ${quoteAlias(col.name)}`); + } + + return parts.join(', '); +} + +/** + * Build the chart config for one source of a multi-source search: the standard + * `buildSearchChartConfig` assembly with the SELECT replaced by the canonical + * aliased column set, so every selected source returns the same result shape. + * + * The caller supplies `orderBy` per source (each source's own timestamp-based + * default) — a shared orderBy is meaningless across schemas, and time-window + * pagination requires the first orderBy term to be the source's timestamp. + */ +export function buildMultiSourceSearchConfig( + source: TSource, + input: Omit, + opts: { + includeDuration?: boolean; + extraColumns?: MultiSourceExtraColumn[]; + } = {}, +): SearchChartConfig { + return buildSearchChartConfig(source, { + ...input, + select: buildMultiSourceSelect(source, opts), + }); +}