[HDX-5126] Apply HAVING/ORDER BY/LIMIT to the composed metric join, not per-series branches

.changeset/composed-outer-clauses.md

@@ -0,0 +1,5 @@
+---
+'@hyperdx/common-utils': patch
+---
+
+HAVING, ORDER BY and LIMIT on multi-series metric charts now apply to the final joined result instead of leaking into each per-series branch. They reference the chart's output columns — operand aliases, formula names/aliases, the ratio column, group-by columns and the time bucket — so a HAVING like `"err rate" > 0.5` filters the joined rows, ORDER BY actually orders the result (previously it was applied per branch and then discarded by the join), and LIMIT/OFFSET paginate one consistent group set across all series.

packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap

@@ -890,6 +890,68 @@ exports[`renderChartConfig multi-series metric charts (composed query) formulas
     ) AS "__hdx_value",toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\`)) GROUP BY ALL ORDER BY \`__hdx_time_bucket\` SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"
 `;
 
+exports[`renderChartConfig multi-series metric charts (composed query) outer HAVING / ORDER BY / LIMIT renders having, orderBy and limit once, on the outer joined statement only 1`] = `
+"SELECT anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 0) AS "avg(metric.alpha)", anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 1) AS "avg(metric.beta)", * EXCEPT (\`__hdx_value\`, \`__hdx_series_idx\`) FROM (SELECT * REPLACE (toFloat64(\`__hdx_value\`) AS \`__hdx_value\`), 0 AS \`__hdx_series_idx\` FROM (WITH Source AS (
+            SELECT
+              *,
+              cityHash64(ScopeAttributes, ResourceAttributes, Attributes) AS AttributesHash
+            FROM default.otel_metrics_gauge
+            WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1739491200000)) AND ((MetricName = 'metric.alpha'))
+          ),Bucketed AS (
+            SELECT
+              toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 hour) AS \`__hdx_time_bucket2\`,
+              AttributesHash,
+              last_value(Value) AS LastValue,
+              any(ScopeAttributes) AS ScopeAttributes,
+              any(ResourceAttributes) AS ResourceAttributes,
+              any(Attributes) AS Attributes,
+              any(ResourceSchemaUrl) AS ResourceSchemaUrl,
+              any(ScopeName) AS ScopeName,
+              any(ScopeVersion) AS ScopeVersion,
+              any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount,
+              any(ScopeSchemaUrl) AS ScopeSchemaUrl,
+              any(ServiceName) AS ServiceName,
+              any(MetricDescription) AS MetricDescription,
+              any(MetricUnit) AS MetricUnit,
+              any(StartTimeUnix) AS StartTimeUnix,
+              any(Flags) AS Flags
+            FROM Source
+            GROUP BY AttributesHash, __hdx_time_bucket2
+            ORDER BY AttributesHash, __hdx_time_bucket2
+          ) SELECT avg(
+      toFloat64OrDefault(toString(LastValue))
+    ) AS "__hdx_value",ServiceName FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY ServiceName) UNION ALL SELECT * REPLACE (toFloat64(\`__hdx_value\`) AS \`__hdx_value\`), 1 AS \`__hdx_series_idx\` FROM (WITH Source AS (
+            SELECT
+              *,
+              cityHash64(ScopeAttributes, ResourceAttributes, Attributes) AS AttributesHash
+            FROM default.otel_metrics_gauge
+            WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1739491200000)) AND ((MetricName = 'metric.beta'))
+          ),Bucketed AS (
+            SELECT
+              toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 hour) AS \`__hdx_time_bucket2\`,
+              AttributesHash,
+              last_value(Value) AS LastValue,
+              any(ScopeAttributes) AS ScopeAttributes,
+              any(ResourceAttributes) AS ResourceAttributes,
+              any(Attributes) AS Attributes,
+              any(ResourceSchemaUrl) AS ResourceSchemaUrl,
+              any(ScopeName) AS ScopeName,
+              any(ScopeVersion) AS ScopeVersion,
+              any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount,
+              any(ScopeSchemaUrl) AS ScopeSchemaUrl,
+              any(ServiceName) AS ServiceName,
+              any(MetricDescription) AS MetricDescription,
+              any(MetricUnit) AS MetricUnit,
+              any(StartTimeUnix) AS StartTimeUnix,
+              any(Flags) AS Flags
+            FROM Source
+            GROUP BY AttributesHash, __hdx_time_bucket2
+            ORDER BY AttributesHash, __hdx_time_bucket2
+          ) SELECT avg(
+      toFloat64OrDefault(toString(LastValue))
+    ) AS "__hdx_value",ServiceName FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY ServiceName)) GROUP BY ALL HAVING "avg(metric.alpha)" > 10 ORDER BY "avg(metric.beta)" DESC LIMIT 5 OFFSET 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"
+`;
+
 exports[`renderChartConfig multi-series metric charts (composed query) pads group columns across gauge and histogram branch classes 1`] = `
 "SELECT anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 0) AS "avg(metric.alpha)", anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 1) AS "quantile(metric.latency)", * EXCEPT (\`__hdx_value\`, \`__hdx_series_idx\`) FROM (SELECT * REPLACE (toFloat64(\`__hdx_value\`) AS \`__hdx_value\`), 0 AS \`__hdx_series_idx\`, [] AS \`group\` FROM (WITH Source AS (
             SELECT

packages/common-utils/src/__tests__/queryChartConfig.int.test.ts

@@ -2028,5 +2028,192 @@ describe('queryChartConfig Integration Tests', () => {
         expect(Number(col(byService.get('svc-b'), 'ratio'))).toBe(0);
       });
     });
+
+    // HAVING / ORDER BY / LIMIT apply to the final joined result and
+    // reference its output columns (HDX-5126) — not each per-series branch,
+    // where the output names don't exist and each series would be
+    // filtered/ordered/truncated independently.
+    //
+    // Fixture recap (grpratio.* grouped by ServiceName, table shape):
+    //   avg(grpratio.err):   svc-a 1.5, svc-b 6, svc-d 5,   svc-c gap
+    //   avg(grpratio.total): svc-a 4.5, svc-b 12, svc-c 8,  svc-d gap
+    describe('outer HAVING / ORDER BY / LIMIT (HDX-5126)', () => {
+      const grpRatioTable = (overrides: Partial<ChartConfigWithOptDateRange>) =>
+        baseConfig({
+          displayType: DisplayType.Table,
+          granularity: undefined,
+          select: [gaugeSelect('grpratio.err'), gaugeSelect('grpratio.total')],
+          groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }],
+          ...overrides,
+        });
+
+      const services = (data: unknown) =>
+        (data as Row[]).map(r => col(r, 'ServiceName'));
+
+      it('filters the joined rows with HAVING on an operand output column', async () => {
+        const result = await runConfig(
+          grpRatioTable({
+            having: '"avg(grpratio.total)" > 5',
+            havingLanguage: 'sql',
+          }),
+        );
+
+        // svc-a (4.5) fails the predicate; svc-d has err data but a NULL
+        // total, and NULL > 5 filters out. A per-branch HAVING could never
+        // drop svc-d — its err branch has no total column to inspect.
+        expect(services(result.data).sort()).toEqual(['svc-b', 'svc-c']);
+      });
+
+      it('filters with HAVING on a formula output column', async () => {
+        const result = await runConfig(
+          grpRatioTable({
+            formulas: [{ expression: 'A / B', alias: 'err rate' }],
+            having: '"err rate" >= 0.5',
+            havingLanguage: 'sql',
+          }),
+        );
+
+        // Rates: svc-a 1.5/4.5≈0.33, svc-b 0.5, svc-c 0/8=0, svc-d gap.
+        expect(services(result.data)).toEqual(['svc-b']);
+        expect(Number(col((result.data as Row[])[0], 'err rate'))).toBeCloseTo(
+          0.5,
+          5,
+        );
+      });
+
+      it('orders by a plain group column across the joined result', async () => {
+        const result = await runConfig(
+          grpRatioTable({
+            orderBy: [{ valueExpression: 'ServiceName', ordering: 'DESC' }],
+          }),
+        );
+
+        expect(services(result.data)).toEqual([
+          'svc-d',
+          'svc-c',
+          'svc-b',
+          'svc-a',
+        ]);
+      });
+
+      it('orders by an output value column and paginates the joined result with LIMIT/OFFSET', async () => {
+        const orderBy = [
+          {
+            valueExpression: '"avg(grpratio.total)"',
+            ordering: 'DESC' as const,
+          },
+        ];
+
+        // Full order: svc-b (12), svc-c (8), svc-a (4.5), svc-d (NULL —
+        // ClickHouse sorts NULLS LAST by default).
+        const page1 = await runConfig(
+          grpRatioTable({ orderBy, limit: { limit: 2 } }),
+        );
+        expect(services(page1.data)).toEqual(['svc-b', 'svc-c']);
+
+        // The second page continues the SAME joined ordering — page windows
+        // are disjoint and the group universe is consistent across series.
+        // (A per-branch LIMIT truncated each series to its own arbitrary
+        // groups before the join, so pages neither aligned nor partitioned.)
+        const page2 = await runConfig(
+          grpRatioTable({ orderBy, limit: { limit: 2, offset: 2 } }),
+        );
+        expect(services(page2.data)).toEqual(['svc-a', 'svc-d']);
+
+        // The joined row is intact on every page: svc-d keeps its err value
+        // and its total gap.
+        const svcD = (page2.data as Row[])[1];
+        expect(Number(col(svcD, 'avg(grpratio.err)'))).toBe(5);
+        expectGap(col(svcD, 'avg(grpratio.total)'));
+      });
+
+      it('orders time-series rows by bucket first, user sort second', async () => {
+        const result = await runConfig(
+          baseConfig({
+            select: [gaugeSelect('grp.one'), gaugeSelect('grp.two')],
+            groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }],
+            orderBy: [{ valueExpression: 'ServiceName', ordering: 'DESC' }],
+          }),
+        );
+
+        // All rows share bucket 0; the user sort breaks the tie in reverse
+        // service order.
+        expect(services(result.data)).toEqual(['svc-c', 'svc-b', 'svc-a']);
+      });
+
+      it('resolves an expression group-by in ORDER BY via its derived output name', async () => {
+        // The passthrough column for an expression group-by keeps its
+        // ClickHouse-derived name. The contract for referencing it from
+        // ORDER BY/HAVING is the (quoted) output name — the raw map-access
+        // expression is not resolvable in the outer scope, where the source
+        // columns no longer exist.
+        const result = await runConfig(
+          grpRatioTable({
+            groupBy: [
+              {
+                aggCondition: '',
+                valueExpression: "ResourceAttributes['service.name']",
+              },
+            ],
+            orderBy: [
+              {
+                valueExpression: `"arrayElement(ResourceAttributes, 'service.name')"`,
+                ordering: 'DESC',
+              },
+            ],
+          }),
+        );
+
+        const DERIVED_NAME = "arrayElement(ResourceAttributes, 'service.name')";
+        expect((result.data as Row[]).map(r => col(r, DERIVED_NAME))).toEqual([
+          'svc-d',
+          'svc-c',
+          'svc-b',
+          'svc-a',
+        ]);
+      });
+
+      it('orders by an aliased expression group-by through the alias', async () => {
+        const result = await runConfig(
+          grpRatioTable({
+            groupBy: [
+              {
+                aggCondition: '',
+                valueExpression: "ResourceAttributes['service.name']",
+                alias: 'service',
+              },
+            ],
+            orderBy: [{ valueExpression: 'service', ordering: 'ASC' }],
+          }),
+        );
+
+        expect((result.data as Row[]).map(r => col(r, 'service'))).toEqual([
+          'svc-a',
+          'svc-b',
+          'svc-c',
+          'svc-d',
+        ]);
+      });
+
+      it('filters and orders the ratio output column', async () => {
+        const result = await runConfig(
+          grpRatioTable({
+            seriesReturnType: 'ratio',
+            having: '"avg(grpratio.err)/avg(grpratio.total)" >= 0.3',
+            havingLanguage: 'sql',
+            orderBy: [
+              {
+                valueExpression: '"avg(grpratio.err)/avg(grpratio.total)"',
+                ordering: 'DESC',
+              },
+            ],
+          }),
+        );
+
+        // Rates: svc-a ≈0.33, svc-b 0.5, svc-c 0, svc-d gap (NULL fails the
+        // predicate).
+        expect(services(result.data)).toEqual(['svc-b', 'svc-a']);
+      });
+    });
   });
 });

packages/common-utils/src/__tests__/renderChartConfig.test.ts

@@ -3966,6 +3966,86 @@ describe('renderChartConfig', () => {
       expect(sql.match(/SETTINGS/g)).toHaveLength(1);
     });
 
+    // HAVING / ORDER BY / LIMIT apply to the final joined result, where the
+    // user-facing output columns exist — never inside a per-series branch,
+    // which would filter/order/truncate each series independently (HDX-5126).
+    describe('outer HAVING / ORDER BY / LIMIT', () => {
+      it('renders having, orderBy and limit once, on the outer joined statement only', async () => {
+        const generatedSql = await renderChartConfig(
+          {
+            ...baseMultiSeriesConfig,
+            displayType: DisplayType.Table,
+            granularity: undefined,
+            groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }],
+            having: '"avg(metric.alpha)" > 10',
+            havingLanguage: 'sql',
+            orderBy: [
+              { valueExpression: '"avg(metric.beta)"', ordering: 'DESC' },
+            ],
+            limit: { limit: 5, offset: 10 },
+          },
+          mockMetadata,
+          querySettings,
+        );
+        const sql = parameterizedQueryToSql(generatedSql);
+        expect(sql).toMatchSnapshot();
+
+        // Exactly one of each user clause in the whole composed statement —
+        // i.e. none leaked into the UNION ALL branches. (Bare ORDER BY also
+        // appears inside the gauge translation's internal CTE scaffolding,
+        // so count the user's exact clause text, not the keyword.)
+        const count = (needle: string) => sql.split(needle).length - 1;
+        expect(count('HAVING "avg(metric.alpha)" > 10')).toBe(1);
+        expect(count('ORDER BY "avg(metric.beta)" DESC')).toBe(1);
+        expect(count('LIMIT 5 OFFSET 10')).toBe(1);
+        expect(count('HAVING')).toBe(1);
+        // And on the outer scope: after the join's GROUP BY ALL, in
+        // HAVING -> ORDER BY -> LIMIT order.
+        const groupByIdx = sql.lastIndexOf('GROUP BY ALL');
+        const havingIdx = sql.indexOf('HAVING "avg(metric.alpha)" > 10');
+        const orderByIdx = sql.indexOf('ORDER BY "avg(metric.beta)" DESC');
+        expect(groupByIdx).toBeGreaterThan(-1);
+        expect(havingIdx).toBeGreaterThan(groupByIdx);
+        expect(orderByIdx).toBeGreaterThan(havingIdx);
+        expect(sql.indexOf('LIMIT 5 OFFSET 10')).toBeGreaterThan(orderByIdx);
+      });
+
+      it('keeps time charts bucket-ordered first, with the user sort as tiebreaker', async () => {
+        const generatedSql = await renderChartConfig(
+          {
+            ...baseMultiSeriesConfig,
+            groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }],
+            orderBy: [{ valueExpression: 'ServiceName', ordering: 'ASC' }],
+          },
+          mockMetadata,
+          querySettings,
+        );
+        const sql = parameterizedQueryToSql(generatedSql);
+        expect(sql).toContain('ORDER BY `__hdx_time_bucket`,ServiceName ASC');
+      });
+
+      it('lets HAVING reference a formula output column', async () => {
+        const generatedSql = await renderChartConfig(
+          {
+            ...baseMultiSeriesConfig,
+            displayType: DisplayType.Table,
+            granularity: undefined,
+            groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }],
+            formulas: [{ expression: 'A / B', alias: 'err rate' }],
+            having: '"err rate" > 0.5',
+            havingLanguage: 'sql',
+          },
+          mockMetadata,
+          querySettings,
+        );
+        const sql = parameterizedQueryToSql(generatedSql);
+        expect(sql.match(/HAVING/g)).toHaveLength(1);
+        expect(sql.indexOf('HAVING "err rate" > 0.5')).toBeGreaterThan(
+          sql.lastIndexOf('GROUP BY ALL'),
+        );
+      });
+    });
+
     // Formula projection over the pivoted per-series columns.
     describe('formulas', () => {
       const pivot = (idx: number) =>