feat(common-utils): canonical SELECT builder for cross-source search

.changeset/multi-source-select-builder.md

@@ -0,0 +1,9 @@
+---
+'@hyperdx/common-utils': minor
+---
+
+Add a builder for search queries that project a canonical, source-independent
+column set. Given a source, it emits that source's semantic expressions
+(timestamp, service, severity/status, body/span name, duration) under shared
+aliases, and pads columns a source doesn't have with NULL, so results from
+different tables share one shape.

packages/common-utils/src/core/__tests__/searchChartConfig.test.ts

@@ -1,6 +1,9 @@
 import {
   ALERT_COUNT_DEFAULT_SELECT,
+  buildMultiSourceSearchConfig,
+  buildMultiSourceSelect,
   buildSearchChartConfig,
+  MULTI_SOURCE_ALIASES,
 } from '@/core/searchChartConfig';
 import { DisplayType, Filter, SourceKind, TSource } from '@/types';
 
@@ -477,3 +480,120 @@ describe('buildSearchChartConfig', () => {
     });
   });
 });
+
+describe('buildMultiSourceSelect', () => {
+  it('projects every canonical alias, so callers can rely on the constants', () => {
+    const select = buildMultiSourceSelect(makeTraceSource(), {
+      includeDuration: true,
+    });
+
+    for (const alias of Object.values(MULTI_SOURCE_ALIASES)) {
+      expect(select).toContain(`AS "${alias}"`);
+    }
+  });
+
+  it('projects the canonical aliases from a Log source semantic expressions', () => {
+    const source = makeLogSource({
+      displayedTimestampValueExpression: 'Timestamp',
+      serviceNameExpression: 'ServiceName',
+      severityTextExpression: 'SeverityText',
+      bodyExpression: 'Body',
+    });
+
+    expect(buildMultiSourceSelect(source)).toBe(
+      'Timestamp AS "__hdx_timestamp", ' +
+        'ServiceName AS "__hdx_service", ' +
+        'SeverityText AS "__hdx_severity", ' +
+        'Body AS "__hdx_body"',
+    );
+  });
+
+  it('falls back to the first timestamp expression and NULL for missing semantics', () => {
+    const source = makeLogSource({
+      timestampValueExpression: 'TimestampTime, Timestamp',
+      implicitColumnExpression: undefined,
+    });
+
+    expect(buildMultiSourceSelect(source)).toBe(
+      'TimestampTime AS "__hdx_timestamp", ' +
+        'NULL AS "__hdx_service", ' +
+        'NULL AS "__hdx_severity", ' +
+        'NULL AS "__hdx_body"',
+    );
+  });
+
+  it('maps Trace sources onto status/span-name and a milliseconds duration', () => {
+    const source = makeTraceSource({
+      serviceNameExpression: 'ServiceName',
+      statusCodeExpression: 'StatusCode',
+      spanNameExpression: 'SpanName',
+      durationExpression: 'Duration',
+      durationPrecision: 9,
+    });
+
+    expect(buildMultiSourceSelect(source, { includeDuration: true })).toBe(
+      'Timestamp AS "__hdx_timestamp", ' +
+        'ServiceName AS "__hdx_service", ' +
+        'StatusCode AS "__hdx_severity", ' +
+        'SpanName AS "__hdx_body", ' +
+        '(Duration)/1e6 AS "__hdx_duration_ms"',
+    );
+  });
+
+  it('projects NULL duration for Log sources when duration is included', () => {
+    const source = makeLogSource({ bodyExpression: 'Body' });
+
+    expect(buildMultiSourceSelect(source, { includeDuration: true })).toContain(
+      'NULL AS "__hdx_duration_ms"',
+    );
+  });
+
+  it('appends extra columns, projecting NULL where a source lacks the column', () => {
+    const source = makeLogSource({ bodyExpression: 'Body' });
+
+    const select = buildMultiSourceSelect(source, {
+      extraColumns: [
+        { name: 'ServiceName', expression: 'ServiceName' },
+        { name: 'StatusCode', expression: null },
+      ],
+    });
+
+    expect(select).toContain('ServiceName AS "ServiceName"');
+    expect(select).toContain('NULL AS "StatusCode"');
+  });
+});
+
+describe('buildMultiSourceSearchConfig', () => {
+  it('keeps the standard search config assembly but swaps in the canonical SELECT', () => {
+    const source = makeLogSource({
+      bodyExpression: 'Body',
+      tableFilterExpression: "ServiceName != 'noisy'",
+    });
+
+    const config = buildMultiSourceSearchConfig(source, {
+      where: 'error',
+      whereLanguage: 'lucene',
+      orderBy: 'TimestampTime DESC',
+    });
+
+    expect(config.select).toBe(buildMultiSourceSelect(source));
+    expect(config.from).toEqual(source.from);
+    expect(config.connection).toBe('conn-1');
+    expect(config.where).toBe('error');
+    expect(config.whereLanguage).toBe('lucene');
+    expect(config.orderBy).toBe('TimestampTime DESC');
+    // Source-level behaviors (e.g. tableFilterExpression) still apply.
+    expect(config.filters).toEqual([
+      { type: 'sql', condition: "ServiceName != 'noisy'" },
+    ]);
+  });
+
+  it('never resolves to defaultTableSelectExpression', () => {
+    const config = buildMultiSourceSearchConfig(makeTraceSource(), {
+      where: '',
+    });
+
+    expect(config.select).not.toContain('SpanName,');
+    expect(config.select).toContain('AS "__hdx_timestamp"');
+  });
+});

packages/common-utils/src/core/searchChartConfig.ts

@@ -1,3 +1,4 @@
+import { getFirstTimestampValueExpression } from '@/core/utils';
 import {
   BuilderChartConfig,
   DateRange,
@@ -185,3 +186,143 @@ export function buildSearchChartConfig(
 
   return config;
 }
+
+/**
+ * Canonical result-column aliases used when searching across multiple sources
+ * at once. Every selected source's SELECT is rewritten to this shape, so the
+ * merged results table can map columns by name regardless of how each source's
+ * underlying schema names them.
+ *
+ * The names are quoted aliases (`expr AS "__hdx_timestamp"`), so ClickHouse
+ * returns them verbatim — unlike raw expressions, which CH may reformat.
+ */
+export const MULTI_SOURCE_ALIASES = {
+  timestamp: '__hdx_timestamp',
+  service: '__hdx_service',
+  severity: '__hdx_severity',
+  body: '__hdx_body',
+  /** Milliseconds; only projected when a Trace source is in the selection. */
+  durationMs: '__hdx_duration_ms',
+} as const;
+
+/**
+ * An extra user-picked column to project alongside the canonical aliases.
+ * `expression` is the per-source SQL expression for the column, or null when
+ * the source has no such column (projected as NULL so every source returns
+ * the same column set).
+ */
+export type MultiSourceExtraColumn = {
+  /** Result column name (used verbatim as the quoted alias). */
+  name: string;
+  expression: string | null;
+};
+
+const quoteAlias = (name: string) => `"${name.replace(/"/g, '\\"')}"`;
+
+/**
+ * Per-source semantic expression for each canonical alias. Mirrors the app's
+ * display helpers (`getDisplayedTimestampValueExpression`, `getEventBody`,
+ * `getDurationMsExpression` in packages/app/src/source.ts) — keep in sync.
+ */
+function multiSourceSemanticExpressions(source: TSource): {
+  timestamp: string;
+  service: string;
+  severity: string;
+  body: string;
+  durationMs: string;
+} {
+  const firstTimestamp = getFirstTimestampValueExpression(
+    source.timestampValueExpression,
+  );
+
+  if (isLogSource(source)) {
+    return {
+      timestamp: source.displayedTimestampValueExpression || firstTimestamp,
+      service: source.serviceNameExpression || 'NULL',
+      severity: source.severityTextExpression || 'NULL',
+      body: source.bodyExpression || source.implicitColumnExpression || 'NULL',
+      durationMs: 'NULL',
+    };
+  }
+
+  if (isTraceSource(source)) {
+    return {
+      timestamp: source.displayedTimestampValueExpression || firstTimestamp,
+      service: source.serviceNameExpression || 'NULL',
+      severity: source.statusCodeExpression || 'NULL',
+      body: source.spanNameExpression || 'NULL',
+      // Match getDurationMsExpression: durationPrecision is the sub-second
+      // digit count (9 = nanoseconds), so /1e(precision-3) yields milliseconds.
+      durationMs: `(${source.durationExpression})/1e${(source.durationPrecision ?? 9) - 3}`,
+    };
+  }
+
+  // Multi-source search only supports Log and Trace sources today; other kinds
+  // still get a valid (if minimal) shape so a stray source can't render SQL
+  // that errors the whole selection.
+  return {
+    timestamp: firstTimestamp,
+    service: 'NULL',
+    severity: 'NULL',
+    body: 'NULL',
+    durationMs: 'NULL',
+  };
+}
+
+/**
+ * Build the canonical aliased SELECT string for one source in a multi-source
+ * search. Exported for tests.
+ */
+export function buildMultiSourceSelect(
+  source: TSource,
+  {
+    includeDuration = false,
+    extraColumns = [],
+  }: {
+    /** Project `__hdx_duration_ms` (set when any selected source is a Trace). */
+    includeDuration?: boolean;
+    extraColumns?: MultiSourceExtraColumn[];
+  } = {},
+): string {
+  const exprs = multiSourceSemanticExpressions(source);
+
+  const parts = [
+    `${exprs.timestamp} AS ${quoteAlias(MULTI_SOURCE_ALIASES.timestamp)}`,
+    `${exprs.service} AS ${quoteAlias(MULTI_SOURCE_ALIASES.service)}`,
+    `${exprs.severity} AS ${quoteAlias(MULTI_SOURCE_ALIASES.severity)}`,
+    `${exprs.body} AS ${quoteAlias(MULTI_SOURCE_ALIASES.body)}`,
+  ];
+  if (includeDuration) {
+    parts.push(
+      `${exprs.durationMs} AS ${quoteAlias(MULTI_SOURCE_ALIASES.durationMs)}`,
+    );
+  }
+  for (const col of extraColumns) {
+    parts.push(`${col.expression ?? 'NULL'} AS ${quoteAlias(col.name)}`);
+  }
+
+  return parts.join(', ');
+}
+
+/**
+ * Build the chart config for one source of a multi-source search: the standard
+ * `buildSearchChartConfig` assembly with the SELECT replaced by the canonical
+ * aliased column set, so every selected source returns the same result shape.
+ *
+ * The caller supplies `orderBy` per source (each source's own timestamp-based
+ * default) — a shared orderBy is meaningless across schemas, and time-window
+ * pagination requires the first orderBy term to be the source's timestamp.
+ */
+export function buildMultiSourceSearchConfig(
+  source: TSource,
+  input: Omit<SearchChartConfigInput, 'select'>,
+  opts: {
+    includeDuration?: boolean;
+    extraColumns?: MultiSourceExtraColumn[];
+  } = {},
+): SearchChartConfig {
+  return buildSearchChartConfig(source, {
+    ...input,
+    select: buildMultiSourceSelect(source, opts),
+  });
+}