diff --git a/.changeset/metric-formula-rendering.md b/.changeset/metric-formula-rendering.md new file mode 100644 index 0000000000..93a4538953 --- /dev/null +++ b/.changeset/metric-formula-rendering.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/common-utils': minor +--- + +Render metric formulas (`formulas` on builder chart configs) in the composed multi-series metric query. Letter-ref expressions like `A / (A + B + C) * 100` compile into the final SELECT projection over the pivoted per-series columns, with ratio-consistent missing-data semantics: a missing operand counts as 0 while a zero or missing division denominator yields NULL (a rendered gap). `showOperandSeries: false` emits only the formula column(s). Works for grouped and ungrouped line, table, and number charts, and single-series charts with a formula now route through the composed query path. diff --git a/.changeset/multi-series-convert-to-sql.md b/.changeset/multi-series-convert-to-sql.md new file mode 100644 index 0000000000..42260c8ef3 --- /dev/null +++ b/.changeset/multi-series-convert-to-sql.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/common-utils': minor +--- + +"Convert to SQL" now supports multi-series, ratio, and formula metric charts. The composed UNION ALL + pivot query is emitted as a macro-based raw-SQL template with a `$__sourceTable()` macro per series branch, instead of returning a "cannot be auto-converted" error. Non-time-series metric charts remain unsupported, matching the existing single-series restriction. diff --git a/packages/common-utils/src/__tests__/__snapshots__/builderToRawSql.test.ts.snap b/packages/common-utils/src/__tests__/__snapshots__/builderToRawSql.test.ts.snap index a571fa49f3..e23a2ed080 100644 --- a/packages/common-utils/src/__tests__/__snapshots__/builderToRawSql.test.ts.snap +++ b/packages/common-utils/src/__tests__/__snapshots__/builderToRawSql.test.ts.snap @@ -252,7 +252,7 @@ ORDER BY $__timeInterval(timestamp) AS \`__hdx_time_bucket\`" `; -exports[`renderBuilderConfigAsSqlTemplate metric charts (single-series only) generates a macro-based template for a single-series gauge metric line chart 1`] = ` +exports[`renderBuilderConfigAsSqlTemplate metric charts generates a macro-based template for a single-series gauge metric line chart 1`] = ` "WITH Source AS ( SELECT @@ -313,7 +313,7 @@ SETTINGS short_circuit_function_evaluation = 'force_enable'" `; -exports[`renderBuilderConfigAsSqlTemplate metric charts (single-series only) generates a macro-based template for a single-series histogram metric line chart 1`] = ` +exports[`renderBuilderConfigAsSqlTemplate metric charts generates a macro-based template for a single-series histogram metric line chart 1`] = ` "WITH source AS ( SELECT @@ -435,7 +435,7 @@ SETTINGS short_circuit_function_evaluation = 'force_enable'" `; -exports[`renderBuilderConfigAsSqlTemplate metric charts (single-series only) generates a macro-based template for a single-series sum metric line chart 1`] = ` +exports[`renderBuilderConfigAsSqlTemplate metric charts generates a macro-based template for a single-series sum metric line chart 1`] = ` "WITH Source AS ( SELECT @@ -552,6 +552,307 @@ ORDER BY $__timeInterval(\`__hdx_time_bucket2\`) AS \`__hdx_time_bucket\`" `; +exports[`renderBuilderConfigAsSqlTemplate metric charts multi-series and formula metric charts compiles formulas into the composed template projection 1`] = ` +"SELECT + anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 0) AS "avg(my.metric)", + anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 1) AS "avg(my.other.metric)", + ( + ( + coalesce( + anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 0), + 0 + ) / nullif( + ( + coalesce( + anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 0), + 0 + ) + coalesce( + anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 1), + 0 + ) + ), + 0 + ) + ) * 100 + ) AS "pct", + * EXCEPT (\`__hdx_value\`, \`__hdx_series_idx\`) +FROM + ( + SELECT + * REPLACE(toFloat64(\`__hdx_value\`) AS \`__hdx_value\`), + 0 AS \`__hdx_series_idx\` + FROM + ( + WITH + Source AS ( + SELECT + *, + cityHash64(ScopeAttributes, ResourceAttributes, Attributes) AS AttributesHash + FROM + $__sourceTable(gauge) + WHERE + ( + TimeUnix >= $__fromTime_ms + AND TimeUnix <= $__toTime_ms + ) + AND ((MetricName = 'my.metric')) + AND $__filters + ), + Bucketed AS ( + SELECT + $__timeInterval(TimeUnix) AS \`__hdx_time_bucket2\`, + AttributesHash, + last_value(Value) AS LastValue, + any(ScopeAttributes) AS ScopeAttributes, + any(ResourceAttributes) AS ResourceAttributes, + any(Attributes) AS Attributes, + any(ResourceSchemaUrl) AS ResourceSchemaUrl, + any(ScopeName) AS ScopeName, + any(ScopeVersion) AS ScopeVersion, + any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount, + any(ScopeSchemaUrl) AS ScopeSchemaUrl, + any(ServiceName) AS ServiceName, + any(MetricDescription) AS MetricDescription, + any(MetricUnit) AS MetricUnit, + any(StartTimeUnix) AS StartTimeUnix, + any(Flags) AS Flags + FROM + Source + GROUP BY + AttributesHash, + __hdx_time_bucket2 + ORDER BY + AttributesHash, + __hdx_time_bucket2 + ) + SELECT + avg(toFloat64OrDefault(toString(LastValue))) AS "__hdx_value", + $__timeInterval(__hdx_time_bucket2) AS \`__hdx_time_bucket\` + FROM + \`Bucketed\` + WHERE + ( + __hdx_time_bucket2 >= $__fromTime_ms + AND __hdx_time_bucket2 <= $__toTime_ms + ) + GROUP BY + $__timeInterval(__hdx_time_bucket2) AS \`__hdx_time_bucket\` + ORDER BY + $__timeInterval(__hdx_time_bucket2) AS \`__hdx_time_bucket\` + ) + UNION ALL + SELECT + * REPLACE(toFloat64(\`__hdx_value\`) AS \`__hdx_value\`), + 1 AS \`__hdx_series_idx\` + FROM + ( + WITH + Source AS ( + SELECT + *, + cityHash64(ScopeAttributes, ResourceAttributes, Attributes) AS AttributesHash + FROM + $__sourceTable(gauge) + WHERE + ( + TimeUnix >= $__fromTime_ms + AND TimeUnix <= $__toTime_ms + ) + AND ((MetricName = 'my.other.metric')) + AND $__filters + ), + Bucketed AS ( + SELECT + $__timeInterval(TimeUnix) AS \`__hdx_time_bucket2\`, + AttributesHash, + last_value(Value) AS LastValue, + any(ScopeAttributes) AS ScopeAttributes, + any(ResourceAttributes) AS ResourceAttributes, + any(Attributes) AS Attributes, + any(ResourceSchemaUrl) AS ResourceSchemaUrl, + any(ScopeName) AS ScopeName, + any(ScopeVersion) AS ScopeVersion, + any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount, + any(ScopeSchemaUrl) AS ScopeSchemaUrl, + any(ServiceName) AS ServiceName, + any(MetricDescription) AS MetricDescription, + any(MetricUnit) AS MetricUnit, + any(StartTimeUnix) AS StartTimeUnix, + any(Flags) AS Flags + FROM + Source + GROUP BY + AttributesHash, + __hdx_time_bucket2 + ORDER BY + AttributesHash, + __hdx_time_bucket2 + ) + SELECT + avg(toFloat64OrDefault(toString(LastValue))) AS "__hdx_value", + $__timeInterval(__hdx_time_bucket2) AS \`__hdx_time_bucket\` + FROM + \`Bucketed\` + WHERE + ( + __hdx_time_bucket2 >= $__fromTime_ms + AND __hdx_time_bucket2 <= $__toTime_ms + ) + GROUP BY + $__timeInterval(__hdx_time_bucket2) AS \`__hdx_time_bucket\` + ORDER BY + $__timeInterval(__hdx_time_bucket2) AS \`__hdx_time_bucket\` + ) + ) +GROUP BY + ALL +ORDER BY + \`__hdx_time_bucket\` +SETTINGS + short_circuit_function_evaluation = 'force_enable'" +`; + +exports[`renderBuilderConfigAsSqlTemplate metric charts multi-series and formula metric charts generates a composed macro-based template for a two-gauge line chart 1`] = ` +"SELECT + anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 0) AS "avg(my.metric)", + anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 1) AS "avg(my.other.metric)", + * EXCEPT (\`__hdx_value\`, \`__hdx_series_idx\`) +FROM + ( + SELECT + * REPLACE(toFloat64(\`__hdx_value\`) AS \`__hdx_value\`), + 0 AS \`__hdx_series_idx\` + FROM + ( + WITH + Source AS ( + SELECT + *, + cityHash64(ScopeAttributes, ResourceAttributes, Attributes) AS AttributesHash + FROM + $__sourceTable(gauge) + WHERE + ( + TimeUnix >= $__fromTime_ms + AND TimeUnix <= $__toTime_ms + ) + AND ((MetricName = 'my.metric')) + AND $__filters + ), + Bucketed AS ( + SELECT + $__timeInterval(TimeUnix) AS \`__hdx_time_bucket2\`, + AttributesHash, + last_value(Value) AS LastValue, + any(ScopeAttributes) AS ScopeAttributes, + any(ResourceAttributes) AS ResourceAttributes, + any(Attributes) AS Attributes, + any(ResourceSchemaUrl) AS ResourceSchemaUrl, + any(ScopeName) AS ScopeName, + any(ScopeVersion) AS ScopeVersion, + any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount, + any(ScopeSchemaUrl) AS ScopeSchemaUrl, + any(ServiceName) AS ServiceName, + any(MetricDescription) AS MetricDescription, + any(MetricUnit) AS MetricUnit, + any(StartTimeUnix) AS StartTimeUnix, + any(Flags) AS Flags + FROM + Source + GROUP BY + AttributesHash, + __hdx_time_bucket2 + ORDER BY + AttributesHash, + __hdx_time_bucket2 + ) + SELECT + avg(toFloat64OrDefault(toString(LastValue))) AS "__hdx_value", + $__timeInterval(__hdx_time_bucket2) AS \`__hdx_time_bucket\` + FROM + \`Bucketed\` + WHERE + ( + __hdx_time_bucket2 >= $__fromTime_ms + AND __hdx_time_bucket2 <= $__toTime_ms + ) + GROUP BY + $__timeInterval(__hdx_time_bucket2) AS \`__hdx_time_bucket\` + ORDER BY + $__timeInterval(__hdx_time_bucket2) AS \`__hdx_time_bucket\` + ) + UNION ALL + SELECT + * REPLACE(toFloat64(\`__hdx_value\`) AS \`__hdx_value\`), + 1 AS \`__hdx_series_idx\` + FROM + ( + WITH + Source AS ( + SELECT + *, + cityHash64(ScopeAttributes, ResourceAttributes, Attributes) AS AttributesHash + FROM + $__sourceTable(gauge) + WHERE + ( + TimeUnix >= $__fromTime_ms + AND TimeUnix <= $__toTime_ms + ) + AND ((MetricName = 'my.other.metric')) + AND $__filters + ), + Bucketed AS ( + SELECT + $__timeInterval(TimeUnix) AS \`__hdx_time_bucket2\`, + AttributesHash, + last_value(Value) AS LastValue, + any(ScopeAttributes) AS ScopeAttributes, + any(ResourceAttributes) AS ResourceAttributes, + any(Attributes) AS Attributes, + any(ResourceSchemaUrl) AS ResourceSchemaUrl, + any(ScopeName) AS ScopeName, + any(ScopeVersion) AS ScopeVersion, + any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount, + any(ScopeSchemaUrl) AS ScopeSchemaUrl, + any(ServiceName) AS ServiceName, + any(MetricDescription) AS MetricDescription, + any(MetricUnit) AS MetricUnit, + any(StartTimeUnix) AS StartTimeUnix, + any(Flags) AS Flags + FROM + Source + GROUP BY + AttributesHash, + __hdx_time_bucket2 + ORDER BY + AttributesHash, + __hdx_time_bucket2 + ) + SELECT + avg(toFloat64OrDefault(toString(LastValue))) AS "__hdx_value", + $__timeInterval(__hdx_time_bucket2) AS \`__hdx_time_bucket\` + FROM + \`Bucketed\` + WHERE + ( + __hdx_time_bucket2 >= $__fromTime_ms + AND __hdx_time_bucket2 <= $__toTime_ms + ) + GROUP BY + $__timeInterval(__hdx_time_bucket2) AS \`__hdx_time_bucket\` + ORDER BY + $__timeInterval(__hdx_time_bucket2) AS \`__hdx_time_bucket\` + ) + ) +GROUP BY + ALL +ORDER BY + \`__hdx_time_bucket\` +SETTINGS + short_circuit_function_evaluation = 'force_enable'" +`; + exports[`renderBuilderConfigAsSqlTemplate series-level WHERE (aggCondition → -If combinators) keeps -If per series but does not push to WHERE when only some series are filtered 1`] = ` "SELECT countIf((ServiceName ILIKE '%api%')), diff --git a/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap b/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap index e2d303a24c..74899add02 100644 --- a/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap +++ b/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap @@ -795,6 +795,101 @@ exports[`renderChartConfig multi-series metric charts (composed query) composes ) AS "__hdx_value",ServiceName,toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY ServiceName,toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\`)) GROUP BY ALL ORDER BY \`__hdx_time_bucket\` SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; +exports[`renderChartConfig multi-series metric charts (composed query) formulas appends the formula column after the operand value columns 1`] = ` +"SELECT anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 0) AS "avg(metric.alpha)", anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 1) AS "avg(metric.beta)", ((coalesce(anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 0), 0) / nullif((coalesce(anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 0), 0) + coalesce(anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 1), 0)), 0)) * 100) AS "Success rate", * EXCEPT (\`__hdx_value\`, \`__hdx_series_idx\`) FROM (SELECT * REPLACE (toFloat64(\`__hdx_value\`) AS \`__hdx_value\`), 0 AS \`__hdx_series_idx\` FROM (WITH Source AS ( + SELECT + *, + cityHash64(ScopeAttributes, ResourceAttributes, Attributes) AS AttributesHash + FROM default.otel_metrics_gauge + WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1739491200000)) AND ((MetricName = 'metric.alpha')) + ),Bucketed AS ( + SELECT + toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS \`__hdx_time_bucket2\`, + AttributesHash, + last_value(Value) AS LastValue, + any(ScopeAttributes) AS ScopeAttributes, + any(ResourceAttributes) AS ResourceAttributes, + any(Attributes) AS Attributes, + any(ResourceSchemaUrl) AS ResourceSchemaUrl, + any(ScopeName) AS ScopeName, + any(ScopeVersion) AS ScopeVersion, + any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount, + any(ScopeSchemaUrl) AS ScopeSchemaUrl, + any(ServiceName) AS ServiceName, + any(MetricDescription) AS MetricDescription, + any(MetricUnit) AS MetricUnit, + any(StartTimeUnix) AS StartTimeUnix, + any(Flags) AS Flags + FROM Source + GROUP BY AttributesHash, __hdx_time_bucket2 + ORDER BY AttributesHash, __hdx_time_bucket2 + ) SELECT avg( + toFloat64OrDefault(toString(LastValue)) + ) AS "__hdx_value",toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\`) UNION ALL SELECT * REPLACE (toFloat64(\`__hdx_value\`) AS \`__hdx_value\`), 1 AS \`__hdx_series_idx\` FROM (WITH Source AS ( + SELECT + *, + cityHash64(ScopeAttributes, ResourceAttributes, Attributes) AS AttributesHash + FROM default.otel_metrics_gauge + WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1739491200000)) AND ((MetricName = 'metric.beta')) + ),Bucketed AS ( + SELECT + toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS \`__hdx_time_bucket2\`, + AttributesHash, + last_value(Value) AS LastValue, + any(ScopeAttributes) AS ScopeAttributes, + any(ResourceAttributes) AS ResourceAttributes, + any(Attributes) AS Attributes, + any(ResourceSchemaUrl) AS ResourceSchemaUrl, + any(ScopeName) AS ScopeName, + any(ScopeVersion) AS ScopeVersion, + any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount, + any(ScopeSchemaUrl) AS ScopeSchemaUrl, + any(ServiceName) AS ServiceName, + any(MetricDescription) AS MetricDescription, + any(MetricUnit) AS MetricUnit, + any(StartTimeUnix) AS StartTimeUnix, + any(Flags) AS Flags + FROM Source + GROUP BY AttributesHash, __hdx_time_bucket2 + ORDER BY AttributesHash, __hdx_time_bucket2 + ) SELECT avg( + toFloat64OrDefault(toString(LastValue)) + ) AS "__hdx_value",toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\`)) GROUP BY ALL ORDER BY \`__hdx_time_bucket\` SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" +`; + +exports[`renderChartConfig multi-series metric charts (composed query) formulas routes a single-series chart with a formula through the composed path 1`] = ` +"SELECT anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 0) AS "avg(metric.alpha)", (coalesce(anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 0), 0) * 100) AS "pct", * EXCEPT (\`__hdx_value\`, \`__hdx_series_idx\`) FROM (SELECT * REPLACE (toFloat64(\`__hdx_value\`) AS \`__hdx_value\`), 0 AS \`__hdx_series_idx\` FROM (WITH Source AS ( + SELECT + *, + cityHash64(ScopeAttributes, ResourceAttributes, Attributes) AS AttributesHash + FROM default.otel_metrics_gauge + WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1739491200000)) AND ((MetricName = 'metric.alpha')) + ),Bucketed AS ( + SELECT + toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS \`__hdx_time_bucket2\`, + AttributesHash, + last_value(Value) AS LastValue, + any(ScopeAttributes) AS ScopeAttributes, + any(ResourceAttributes) AS ResourceAttributes, + any(Attributes) AS Attributes, + any(ResourceSchemaUrl) AS ResourceSchemaUrl, + any(ScopeName) AS ScopeName, + any(ScopeVersion) AS ScopeVersion, + any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount, + any(ScopeSchemaUrl) AS ScopeSchemaUrl, + any(ServiceName) AS ServiceName, + any(MetricDescription) AS MetricDescription, + any(MetricUnit) AS MetricUnit, + any(StartTimeUnix) AS StartTimeUnix, + any(Flags) AS Flags + FROM Source + GROUP BY AttributesHash, __hdx_time_bucket2 + ORDER BY AttributesHash, __hdx_time_bucket2 + ) SELECT avg( + toFloat64OrDefault(toString(LastValue)) + ) AS "__hdx_value",toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\`)) GROUP BY ALL ORDER BY \`__hdx_time_bucket\` SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" +`; + exports[`renderChartConfig multi-series metric charts (composed query) pads group columns across gauge and histogram branch classes 1`] = ` "SELECT anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 0) AS "avg(metric.alpha)", anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = 1) AS "quantile(metric.latency)", * EXCEPT (\`__hdx_value\`, \`__hdx_series_idx\`) FROM (SELECT * REPLACE (toFloat64(\`__hdx_value\`) AS \`__hdx_value\`), 0 AS \`__hdx_series_idx\`, [] AS \`group\` FROM (WITH Source AS ( SELECT diff --git a/packages/common-utils/src/__tests__/builderToRawSql.test.ts b/packages/common-utils/src/__tests__/builderToRawSql.test.ts index 687e48278c..b516531c84 100644 --- a/packages/common-utils/src/__tests__/builderToRawSql.test.ts +++ b/packages/common-utils/src/__tests__/builderToRawSql.test.ts @@ -208,7 +208,7 @@ describe('renderBuilderConfigAsSqlTemplate', () => { expect(sql).toMatchSnapshot(); }); - describe('metric charts (single-series only)', () => { + describe('metric charts', () => { const metricTables = { gauge: 'otel_metrics_gauge', histogram: 'otel_metrics_histogram', @@ -348,25 +348,135 @@ describe('renderBuilderConfigAsSqlTemplate', () => { }, ); - it('returns null for a multi-series metric chart', async () => { - const config = metricLineConfig(MetricsDataType.Gauge); - const sql = await renderBuilderConfigAsSqlTemplate( - { + // Multi-series (and formula/ratio) metric charts render as one composed + // UNION ALL + pivot statement whose branches each carry their own + // $__sourceTable(metricType) and time macros, so they convert to a + // raw-SQL template like any other metric chart. + describe('multi-series and formula metric charts', () => { + const secondSelect = ( + metricType: MetricsDataType, + aggFn = 'avg', + ): any => ({ + aggFn, + aggCondition: '', + valueExpression: 'Value', + metricType, + metricName: 'my.other.metric', + }); + + const multiSeriesConfig = ( + secondType: MetricsDataType, + secondAggFn = 'avg', + ): ChartConfigWithOptDateRange => { + const config = metricLineConfig(MetricsDataType.Gauge); + return { ...config, select: [ ...((config as any).select as any[]), - { - aggFn: 'sum', - aggCondition: '', - valueExpression: 'Value', - metricType: MetricsDataType.Gauge, - metricName: 'my.other.metric', - }, + secondSelect(secondType, secondAggFn), ], - }, - mockMetadata, - ); - expect(sql).toBeNull(); + }; + }; + + it('generates a composed macro-based template for a two-gauge line chart', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + multiSeriesConfig(MetricsDataType.Gauge), + mockMetadata, + ); + expect(sql).not.toBeNull(); + // Branches compose via UNION ALL and pivot on the series index. + expect(sql).toContain('UNION ALL'); + expect(sql).toContain('anyOrNullIf'); + // Each branch emits its own typed source-table macro, no hardcoding. + expect(sql!.match(/\$__sourceTable\(gauge\)/g)).toHaveLength(2); + expect(sql).not.toContain('otel_metrics_gauge'); + // Still wired to the dashboard time range and granularity. + expect(sql).toContain('$__fromTime_ms'); + expect(sql).toContain('$__toTime_ms'); + expect(sql).toContain('$__timeInterval'); + expect(sql).not.toContain('HYPERDX_PARAM_'); + expect(sql).not.toMatch(/INTERVAL 1 minute/i); + expect(sql).toMatchSnapshot(); + }); + + it('emits a typed source-table macro per branch for mixed metric types', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + multiSeriesConfig(MetricsDataType.Sum, 'sum'), + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql).toContain('$__sourceTable(gauge)'); + expect(sql).toContain('$__sourceTable(sum)'); + expect(sql).not.toContain('otel_metrics_gauge'); + expect(sql).not.toContain('otel_metrics_sum'); + expect(sql).not.toContain('HYPERDX_PARAM_'); + }); + + it('compiles formulas into the composed template projection', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + { + ...multiSeriesConfig(MetricsDataType.Gauge), + formulas: [{ expression: 'A / (A + B) * 100', alias: 'pct' }], + }, + mockMetadata, + ); + expect(sql).not.toBeNull(); + // The compiled formula projects over the pivot expressions with the + // ratio-consistent coalesce/nullif semantics. + expect(sql).toContain('AS "pct"'); + expect(sql).toContain('coalesce'); + expect(sql).toContain('nullif'); + expect(sql).not.toContain('HYPERDX_PARAM_'); + expect(sql).toMatchSnapshot(); + }); + + it('renders a ratio chart as a SQL-side division in the template', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + { + ...multiSeriesConfig(MetricsDataType.Gauge), + seriesReturnType: 'ratio', + }, + mockMetadata, + ); + expect(sql).not.toBeNull(); + expect(sql).toContain('AS "avg(my.metric)/avg(my.other.metric)"'); + expect(sql).toContain('nullif'); + expect(sql).not.toContain('HYPERDX_PARAM_'); + }); + + it('emits $__filters once per branch source CTE, never in the outer pivot', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + multiSeriesConfig(MetricsDataType.Gauge), + mockMetadata, + ); + expect(sql).not.toBeNull(); + // One per branch: each branch filters at its own source. + expect(sql!.match(/\$__filters/g)).toHaveLength(2); + // The outer pivot projection (everything before the first branch's + // source CTE) and the trailing GROUP BY/ORDER BY carry no filters — + // they read internal columns the dashboard filters don't know. + expect(sql!.slice(0, sql!.indexOf('WITH'))).not.toContain('$__filters'); + expect(sql!.slice(sql!.lastIndexOf('GROUP BY ALL'))).not.toContain( + '$__filters', + ); + }); + + it('resolves each branch source-table macro through replaceMacros', async () => { + const sql = await renderBuilderConfigAsSqlTemplate( + multiSeriesConfig(MetricsDataType.Sum, 'sum'), + mockMetadata, + ); + expect(sql).not.toBeNull(); + + const expanded = replaceMacros({ + sqlTemplate: sql!, + from: { databaseName: 'default', tableName: '' }, + metricTables, + }); + expect(expanded).not.toContain('$__'); + expect(expanded).toContain('`default`.`otel_metrics_gauge`'); + expect(expanded).toContain('`default`.`otel_metrics_sum`'); + }); }); it('returns null for a non-time-series metric chart', async () => { @@ -453,10 +563,12 @@ describe('renderBuilderConfigAsSqlTemplate', () => { granularity: '1 minute', }; - it('reports a specific reason for a multi-series metric chart', async () => { + it('reports the time-series-only reason for a multi-series metric table chart', async () => { const result = await renderBuilderConfigAsSqlTemplateResult( { ...metricConfig, + displayType: DisplayType.Table, + granularity: undefined, select: [ ...(metricConfig.select as any[]), { @@ -472,7 +584,8 @@ describe('renderBuilderConfigAsSqlTemplate', () => { ); expect(result).toEqual({ isError: true, - error: 'Multi-series metric charts cannot be auto-converted to SQL.', + error: + 'Metric charts can only be auto-converted to SQL for time series display types.', }); }); diff --git a/packages/common-utils/src/__tests__/formula.test.ts b/packages/common-utils/src/__tests__/formula.test.ts index 6282d3ed99..9c63788671 100644 --- a/packages/common-utils/src/__tests__/formula.test.ts +++ b/packages/common-utils/src/__tests__/formula.test.ts @@ -1,4 +1,5 @@ import { + compileFormulaAst, FormulaAst, indexToSeriesRef, MAX_FORMULA_DEPTH, @@ -463,3 +464,59 @@ describe('validateFormula', () => { expect(errors).toMatchObject([{ code: 'empty-expression' }]); }); }); + +describe('compileFormulaAst', () => { + /** Compile an expression against v0/v1/... series value placeholders. */ + const compile = (expression: string): string => { + const parsed = expectOk(parseFormula(expression)); + return compileFormulaAst(parsed.ast, index => `v${index}`); + }; + + it('compiles a series ref to a 0-coalesced value expression', () => { + expect(compile('A')).toBe('coalesce(v0, 0)'); + expect(compile('C')).toBe('coalesce(v2, 0)'); + }); + + it('compiles numeric literals as-is', () => { + expect(compile('A + 2')).toBe('(coalesce(v0, 0) + 2)'); + expect(compile('A * 0.5')).toBe('(coalesce(v0, 0) * 0.5)'); + }); + + it('wraps division denominators in nullif so /0 and /missing read NULL', () => { + expect(compile('A / B')).toBe( + '(coalesce(v0, 0) / nullif(coalesce(v1, 0), 0))', + ); + }); + + it('nullif-wraps compound denominators, not just bare refs', () => { + expect(compile('A / (B + C)')).toBe( + '(coalesce(v0, 0) / nullif((coalesce(v1, 0) + coalesce(v2, 0)), 0))', + ); + }); + + it('compiles the motivating success-rate example', () => { + expect(compile('A / (A + B + C) * 100')).toBe( + '((coalesce(v0, 0) / nullif(((coalesce(v0, 0) + coalesce(v1, 0)) + coalesce(v2, 0)), 0)) * 100)', + ); + }); + + it('parenthesizes to the parsed precedence, not textual order', () => { + expect(compile('A + B * C')).toBe( + '(coalesce(v0, 0) + (coalesce(v1, 0) * coalesce(v2, 0)))', + ); + expect(compile('(A + B) * C')).toBe( + '((coalesce(v0, 0) + coalesce(v1, 0)) * coalesce(v2, 0))', + ); + }); + + it('compiles unary minus', () => { + expect(compile('-A')).toBe('(-coalesce(v0, 0))'); + expect(compile('B - -A')).toBe('(coalesce(v1, 0) - (-coalesce(v0, 0)))'); + }); + + it('nested divisions each get their own nullif guard', () => { + expect(compile('A / B / C')).toBe( + '((coalesce(v0, 0) / nullif(coalesce(v1, 0), 0)) / nullif(coalesce(v2, 0), 0))', + ); + }); +}); diff --git a/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts b/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts index 7e04b06437..0493df923a 100644 --- a/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts +++ b/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts @@ -1267,6 +1267,15 @@ describe('queryChartConfig Integration Tests', () => { gaugeRow('tbl.one', insertTs(0), 'svc-a', 10), gaugeRow('tbl.one', insertTs(0), 'svc-b', 20), gaugeRow('tbl.two', insertTs(0), 'svc-a', 100), + // Formula motivating example: + // success / (success + error + fsi) * 100. Bucket 0 has all three + // operands; bucket 1 has only a zero success (zero denominator); + // bucket 2 has only errors (missing numerator). + gaugeRow('form.success', insertTs(0), 'svc-a', 90), + gaugeRow('form.error', insertTs(0), 'svc-a', 8), + gaugeRow('form.fsi', insertTs(0), 'svc-a', 2), + gaugeRow('form.success', insertTs(1), 'svc-a', 0), + gaugeRow('form.error', insertTs(2), 'svc-a', 5), // Grouped gauge+histogram mix. gaugeRow('grpmix.gauge', insertTs(1), 'svc-a', 42), ], @@ -1825,5 +1834,199 @@ describe('queryChartConfig Integration Tests', () => { expect(col(data[0], 'avg(tbl.one)')).toBe(15); expect(col(data[0], 'avg(tbl.two)')).toBe(100); }); + + // Formulas compile into the composed query's final projection, reusing + // the same fixtures as the merge baseline above. + describe('formulas', () => { + it('computes the motivating success-rate example: A / (A + B + C) * 100', async () => { + const result = await runConfig( + baseConfig({ + select: [ + gaugeSelect('form.success'), + gaugeSelect('form.error'), + gaugeSelect('form.fsi'), + ], + formulas: [ + { expression: 'A / (A + B + C) * 100', alias: 'Success rate' }, + ], + }), + ); + + // Meta contract: operand value columns first, in select order, then + // the formula column — all numeric — ahead of the bucket column. + expectNumericValueColumns(result.meta, [ + 'avg(form.success)', + 'avg(form.error)', + 'avg(form.fsi)', + 'Success rate', + ]); + + const rows = rowsByBucket(result.data); + expect([...rows.keys()].sort()).toEqual([ + bucket(0), + bucket(1), + bucket(2), + ]); + + // All operands present: 90 / (90 + 8 + 2) * 100. + expect(Number(col(rows.get(bucket(0)), 'Success rate'))).toBeCloseTo( + 90, + 5, + ); + // Zero success and nothing else: denominator 0 -> gap, not 0 or error. + expectGap(col(rows.get(bucket(1)), 'Success rate')); + // Missing success counts as 0: 0 / (0 + 5 + 0) * 100 = 0. + expect(Number(col(rows.get(bucket(2)), 'Success rate'))).toBe(0); + }); + + it('matches the ratio projection semantics for A / B (0-for-missing-numerator, gap-for-zero/missing-denominator)', async () => { + const result = await runConfig( + baseConfig({ + select: [gaugeSelect('ratio.err'), gaugeSelect('ratio.total')], + formulas: [{ expression: 'A / B', alias: 'err rate' }], + showOperandSeries: false, + }), + ); + + const rows = rowsByBucket(result.data); + expect(rows.size).toBe(4); + + // Same fixture and expectations as the seriesReturnType: 'ratio' + // test above — the formula path must be drop-in consistent. + expect(Number(col(rows.get(bucket(0)), 'err rate'))).toBeCloseTo( + 0.5, + 5, + ); + expect(Number(col(rows.get(bucket(1)), 'err rate'))).toBe(0); + expectGap(col(rows.get(bucket(2)), 'err rate')); + expectGap(col(rows.get(bucket(3)), 'err rate')); + }); + + it('computes a formula over mixed gauge and sum (increase) operands', async () => { + const result = await runConfig( + baseConfig({ + select: [gaugeSelect('mix.cpu'), increaseSelect('mix.requests')], + formulas: [{ expression: 'A + B', alias: 'combined' }], + }), + ); + + expectNumericValueColumns(result.meta, [ + 'avg(mix.cpu)', + 'increase(mix.requests)', + 'combined', + ]); + + const rows = rowsByBucket(result.data); + // Gauge 1 + increase 0 at bucket 0; missing gauge counts as 0 at + // bucket 1; both present at bucket 2. + expect(Number(col(rows.get(bucket(0)), 'combined'))).toBe(1); + expect(Number(col(rows.get(bucket(1)), 'combined'))).toBe(9); + expect(Number(col(rows.get(bucket(2)), 'combined'))).toBe(13); + }); + + it('computes a grouped formula per (bucket, group) row', async () => { + const result = await runConfig( + baseConfig({ + select: [gaugeSelect('grp.one'), gaugeSelect('grp.two')], + formulas: [{ expression: 'A + B', alias: 'both' }], + groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }], + }), + ); + + expectNumericValueColumns(result.meta, [ + 'avg(grp.one)', + 'avg(grp.two)', + 'both', + ]); + + const rows = rowsByBucketAndGroup(result.data, 'ServiceName'); + expect(rows.size).toBe(3); + expect(Number(col(rows.get(`${bucket(0)}|svc-a`), 'both'))).toBe(11); + // One-sided groups: the missing operand contributes 0. + expect(Number(col(rows.get(`${bucket(0)}|svc-b`), 'both'))).toBe(2); + expect(Number(col(rows.get(`${bucket(0)}|svc-c`), 'both'))).toBe(30); + }); + + it('drops the operand columns from meta and rows when showOperandSeries is false', async () => { + const result = await runConfig( + baseConfig({ + select: [gaugeSelect('grp.one'), gaugeSelect('grp.two')], + formulas: [{ expression: 'A + B', alias: 'both' }], + showOperandSeries: false, + groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }], + }), + ); + + // The formula column leads the meta; the operands are gone but the + // group/bucket passthrough columns survive. + expectNumericValueColumns(result.meta, ['both']); + const metaNames = result.meta?.map(m => m.name) ?? []; + expect(metaNames).not.toContain('avg(grp.one)'); + expect(metaNames).not.toContain('avg(grp.two)'); + expect(metaNames).toContain('ServiceName'); + + const rows = rowsByBucketAndGroup(result.data, 'ServiceName'); + expect(rows.size).toBe(3); + expect(Number(col(rows.get(`${bucket(0)}|svc-a`), 'both'))).toBe(11); + }); + + it('computes a single-series formula (composed path with one branch)', async () => { + const result = await runConfig( + baseConfig({ + select: [gaugeSelect('gap.one')], + formulas: [{ expression: 'A * 100', alias: 'pct' }], + }), + ); + + expectNumericValueColumns(result.meta, ['avg(gap.one)', 'pct']); + + const rows = rowsByBucket(result.data); + expect(rows.size).toBe(2); + expect(Number(col(rows.get(bucket(0)), 'pct'))).toBe(1000); + expect(Number(col(rows.get(bucket(1)), 'pct'))).toBe(2000); + }); + + it('computes formulas for number-shape (ungrouped, no time bucket) charts', async () => { + const result = await runConfig( + baseConfig({ + displayType: DisplayType.Number, + granularity: undefined, + select: [gaugeSelect('tbl.one'), gaugeSelect('tbl.two')], + formulas: [{ expression: 'A / B * 100', alias: 'pct' }], + showOperandSeries: false, + }), + ); + + expectNumericValueColumns(result.meta, ['pct']); + const data = result.data as Row[]; + expect(data).toHaveLength(1); + // avg(tbl.one) = 15, avg(tbl.two) = 100. + expect(Number(col(data[0], 'pct'))).toBeCloseTo(15, 5); + }); + + it('computes formulas for grouped table-shape charts', async () => { + const result = await runConfig( + baseConfig({ + displayType: DisplayType.Table, + granularity: undefined, + select: [gaugeSelect('tbl.one'), gaugeSelect('tbl.two')], + formulas: [{ expression: 'B / A', alias: 'ratio' }], + groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }], + }), + ); + + expectNumericValueColumns(result.meta, [ + 'avg(tbl.one)', + 'avg(tbl.two)', + 'ratio', + ]); + + const data = result.data as Row[]; + const byService = new Map(data.map(r => [col(r, 'ServiceName'), r])); + expect(Number(col(byService.get('svc-a'), 'ratio'))).toBeCloseTo(10, 5); + // svc-b has no tbl.two rows: 0 / 20 = 0. + expect(Number(col(byService.get('svc-b'), 'ratio'))).toBe(0); + }); + }); }); }); diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 0e7e074497..5ca57d18cb 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -3965,5 +3965,175 @@ describe('renderChartConfig', () => { expect(sql).not.toContain('`__hdx_time_bucket`'); expect(sql.match(/SETTINGS/g)).toHaveLength(1); }); + + // Formula projection over the pivoted per-series columns. + describe('formulas', () => { + const pivot = (idx: number) => + `anyOrNullIf(\`__hdx_value\`, \`__hdx_series_idx\` = ${idx})`; + + it('appends the formula column after the operand value columns', async () => { + const generatedSql = await renderChartConfig( + { + ...baseMultiSeriesConfig, + formulas: [ + { expression: 'A / (A + B) * 100', alias: 'Success rate' }, + ], + }, + mockMetadata, + querySettings, + ); + const sql = parameterizedQueryToSql(generatedSql); + expect(sql).toMatchSnapshot(); + + // Operand series pivot under their user-facing aliases, in select + // order, followed by the compiled formula column. + expect(sql).toContain(`${pivot(0)} AS "avg(metric.alpha)"`); + expect(sql).toContain(`${pivot(1)} AS "avg(metric.beta)"`); + const formulaSql = `((coalesce(${pivot(0)}, 0) / nullif((coalesce(${pivot(0)}, 0) + coalesce(${pivot(1)}, 0)), 0)) * 100) AS "Success rate"`; + expect(sql).toContain(formulaSql); + expect(sql.indexOf('AS "avg(metric.beta)"')).toBeLessThan( + sql.indexOf('AS "Success rate"'), + ); + }); + + it('emits only the formula column when showOperandSeries is false', async () => { + const generatedSql = await renderChartConfig( + { + ...baseMultiSeriesConfig, + formulas: [{ expression: 'A / B', alias: 'ratio' }], + showOperandSeries: false, + }, + mockMetadata, + querySettings, + ); + const sql = parameterizedQueryToSql(generatedSql); + expect(sql).not.toContain('AS "avg(metric.alpha)"'); + expect(sql).not.toContain('AS "avg(metric.beta)"'); + expect(sql).toContain( + `(coalesce(${pivot(0)}, 0) / nullif(coalesce(${pivot(1)}, 0), 0)) AS "ratio"`, + ); + // The passthrough bucket column + grouping survive. + expect(sql).toContain( + '* EXCEPT (`__hdx_value`, `__hdx_series_idx`) FROM', + ); + expect(sql).toContain('GROUP BY ALL ORDER BY `__hdx_time_bucket`'); + }); + + it('names an alias-less formula column by its expression text', async () => { + const generatedSql = await renderChartConfig( + { + ...baseMultiSeriesConfig, + formulas: [{ expression: 'A + B' }], + }, + mockMetadata, + querySettings, + ); + const sql = parameterizedQueryToSql(generatedSql); + expect(sql).toContain('AS "A + B"'); + }); + + it('routes a single-series chart with a formula through the composed path', async () => { + const generatedSql = await renderChartConfig( + { + ...baseMultiSeriesConfig, + select: [gaugeSelect('metric.alpha')], + formulas: [{ expression: 'A * 100', alias: 'pct' }], + }, + mockMetadata, + querySettings, + ); + const sql = parameterizedQueryToSql(generatedSql); + expect(sql).toMatchSnapshot(); + expect(sql).toContain(`(coalesce(${pivot(0)}, 0) * 100) AS "pct"`); + expect(sql).toContain( + 'SELECT * REPLACE (toFloat64(`__hdx_value`) AS `__hdx_value`), 0 AS `__hdx_series_idx`', + ); + expect(sql).not.toContain('UNION ALL'); + }); + + it('takes precedence over seriesReturnType ratio', async () => { + const generatedSql = await renderChartConfig( + { + ...baseMultiSeriesConfig, + seriesReturnType: 'ratio', + formulas: [{ expression: 'B / A', alias: 'inverse' }], + }, + mockMetadata, + querySettings, + ); + const sql = parameterizedQueryToSql(generatedSql); + expect(sql).toContain('AS "inverse"'); + expect(sql).not.toContain('AS "avg(metric.alpha)/avg(metric.beta)"'); + }); + + it('suffixes a formula name colliding with an operand alias', async () => { + const generatedSql = await renderChartConfig( + { + ...baseMultiSeriesConfig, + formulas: [{ expression: 'A + B', alias: 'avg(metric.alpha)' }], + }, + mockMetadata, + querySettings, + ); + const sql = parameterizedQueryToSql(generatedSql); + expect(sql).toContain('AS "avg(metric.alpha)"'); + // Formula column index continues after the select entries (2). + expect(sql).toContain('AS "avg(metric.alpha)__2"'); + }); + + it('escapes double quotes in formula and operand column names', async () => { + const generatedSql = await renderChartConfig( + { + ...baseMultiSeriesConfig, + select: [ + { ...gaugeSelect('metric.alpha'), alias: 'operand "quoted"' }, + gaugeSelect('metric.beta'), + ], + formulas: [{ expression: 'A / B', alias: 'bad"name' }], + }, + mockMetadata, + querySettings, + ); + const sql = parameterizedQueryToSql(generatedSql); + // ClickHouse escapes a double quote inside a double-quoted + // identifier by doubling it — a raw interpolation would terminate + // the identifier early (AS "bad"name") and fail to parse. + expect(sql).toContain('AS "bad""name"'); + expect(sql).toContain('AS "operand ""quoted"""'); + expect(sql).not.toContain('AS "bad"name"'); + }); + + it('escapes double quotes in the ratio column label', async () => { + const generatedSql = await renderChartConfig( + { + ...baseMultiSeriesConfig, + select: [ + { ...gaugeSelect('metric.alpha'), alias: 'err"s' }, + { ...gaugeSelect('metric.beta'), alias: 'total' }, + ], + seriesReturnType: 'ratio', + }, + mockMetadata, + querySettings, + ); + const sql = parameterizedQueryToSql(generatedSql); + expect(sql).toContain('AS "err""s/total"'); + }); + + it('throws a structured error for an invalid persisted formula', async () => { + await expect( + renderChartConfig( + { + ...baseMultiSeriesConfig, + formulas: [{ expression: 'A / C' }], + }, + mockMetadata, + querySettings, + ), + ).rejects.toThrow( + 'Invalid formula "A / C": Unknown series "C" — this chart only has series A through B', + ); + }); + }); }); }); diff --git a/packages/common-utils/src/core/builderToRawSql.ts b/packages/common-utils/src/core/builderToRawSql.ts index 8c14e1d7c9..fe90e7babe 100644 --- a/packages/common-utils/src/core/builderToRawSql.ts +++ b/packages/common-utils/src/core/builderToRawSql.ts @@ -73,9 +73,9 @@ export type RenderedSqlTemplate = * * On success returns `{ sql }`. When the config can't be represented as a * single raw-SQL chart it returns `{ error }` with a user-facing reason — - * non-builder configs, multi-series or non-time-series metric charts, string - * selects (Search / EventPatterns), display types without raw-SQL support, or - * a missing source — so callers can surface the message without re-deriving it. + * non-builder configs, non-time-series metric charts, string selects + * (Search / EventPatterns), display types without raw-SQL support, or a + * missing source — so callers can surface the message without re-deriving it. */ export async function renderBuilderConfigAsSqlTemplate( config: ChartConfigWithOptDateRange, @@ -88,22 +88,14 @@ export async function renderBuilderConfigAsSqlTemplate( }; } - // Multi-series metric charts compose per-series branch queries into a - // UNION ALL + pivot statement (see renderMultiSeriesMetricChartConfig). - // That composed shape hasn't been wired into the raw-SQL template macros - // yet ($__sourceTable(metricType) per branch), so only single-series metric - // charts can be converted to a raw-SQL query for now. - const isMetric = config.metricTables != null; - if (isMetric && Array.isArray(config.select) && config.select.length > 1) { - return { - isError: true, - error: 'Multi-series metric charts cannot be auto-converted to SQL.', - }; - } - // A concrete source table is required for non-metric charts; metric charts // resolve their table from metricTables via the $__sourceTable(metricType) - // macro, so they only need the database. + // macro, so they only need the database. Multi-series (and formula/ratio) + // metric charts render as one composed UNION ALL + pivot statement (see + // renderMultiSeriesMetricChartConfig) whose branches each emit their own + // $__sourceTable(metricType) macro, so they convert like any other metric + // chart. + const isMetric = config.metricTables != null; if (!config.from?.databaseName || (!isMetric && !config.from?.tableName)) { return { isError: true, diff --git a/packages/common-utils/src/core/formula.ts b/packages/common-utils/src/core/formula.ts index 2fb6273be8..0230a3ee93 100644 --- a/packages/common-utils/src/core/formula.ts +++ b/packages/common-utils/src/core/formula.ts @@ -510,3 +510,49 @@ const findRefPosition = (expression: string, ref: string): number => { const position = expression.indexOf(ref); return position >= 0 ? position : 0; }; + +// ─── SQL compiler ──────────────────────────────────────────────────────────── + +/** + * Compile a validated formula AST into a SQL expression over per-series value + * expressions. + * + * `resolveSeriesRef` maps a zero-based `select` index to the SQL expression + * producing that series' value — for the composed multi-series metric query + * this is the pivot expression `anyOrNullIf(__hdx_value, __hdx_series_idx = + * i)`, which is NULL when the series has no row at the joined key. + * + * Missing-data semantics mirror the existing `seriesReturnType: 'ratio'` + * projection so a formula `A / B` behaves exactly like the ratio toggle: + * - a series ref compiles to `coalesce(, 0)` — a missing operand + * counts as 0 (a zero-error group reads 0%, not "no data"); + * - a division denominator is wrapped in `nullif(, 0)` — a + * missing or zero denominator makes the quotient NULL, which renders as a + * gap rather than 0 or an error. + * + * The compiler only walks the validated AST (produced by `parseFormula` / + * `validateFormula`); user input is never spliced into SQL as raw text. + */ +export const compileFormulaAst = ( + ast: FormulaAst, + resolveSeriesRef: (index: number) => string, +): string => { + switch (ast.type) { + case 'number': + // JS number stringification of a parsed non-negative literal is plain + // digits / decimal notation, valid as a SQL numeric literal. + return String(ast.value); + case 'seriesRef': + return `coalesce(${resolveSeriesRef(ast.index)}, 0)`; + case 'unary': + return `(-${compileFormulaAst(ast.operand, resolveSeriesRef)})`; + case 'binary': { + const left = compileFormulaAst(ast.left, resolveSeriesRef); + const right = compileFormulaAst(ast.right, resolveSeriesRef); + if (ast.op === '/') { + return `(${left} / nullif(${right}, 0))`; + } + return `(${left} ${ast.op} ${right})`; + } + } +}; diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index 1c35ca8c23..f27240d9b3 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -4,6 +4,7 @@ import SqlString from 'sqlstring'; import { ChSql, chSql, concatChSql, wrapChSqlIfNotEmpty } from '@/clickhouse'; import { stripTypeWrappers } from '@/core/eventDeltas'; +import { compileFormulaAst, FormulaAst, validateFormula } from '@/core/formula'; import { GROUP_ALIAS, translateExponentialHistogram, @@ -120,6 +121,11 @@ export const isMetricChartConfig = ( return chartConfig.metricTables != null; }; +/** Whether the config carries at least one metric formula. */ +export const hasMetricFormulas = ( + chartConfig: BuilderChartConfigWithOptDateRange, +): boolean => (chartConfig.formulas?.length ?? 0) > 0; + // TODO: apply this to all chart configs export const setChartSelectsAlias = ( config: BuilderChartConfigWithOptDateRange, @@ -151,6 +157,16 @@ export const setChartSelectsAlias = ( const MULTI_SERIES_VALUE_ALIAS = '__hdx_value'; const MULTI_SERIES_IDX_ALIAS = '__hdx_series_idx'; +/** + * Render a user-facing output column name as a ClickHouse double-quoted + * identifier. User aliases (and metric names, which flow into the default + * aliases) can contain double quotes; ClickHouse escapes them by doubling, + * so `bad"name` becomes "bad""name" instead of terminating the identifier + * early. Escaping happens only at SQL-emission time — collision dedup and + * the meta column names consumers see keep the raw name. + */ +const quotedColumnName = (name: string) => `"${name.replace(/"/g, '""')}"`; + // Histogram translations bake the group-by dimensions into a single Array // column named GROUP_ALIAS instead of projecting them as individual columns // (see translateHistogram), so grouped histogram rows never share a merge key @@ -2265,6 +2281,14 @@ export async function renderRawSqlChartConfig( * columns with a single "/" column: a missing numerator counts * as 0, a missing or zero denominator yields NULL (a gap), and ratioMode * 'share_of_total' divides by the per-bucket denominator total; + * - `formulas` append one derived column per formula after the + * operand value columns, compiled from the validated letter-ref AST over + * the pivot expressions with the same missing-data semantics as the ratio + * projection (see compileFormulaAst). `showOperandSeries: false` drops the + * raw operand columns so only the formula column(s) remain (still first, + * ahead of the group/bucket passthrough columns). When formulas are + * present they take precedence over `seriesReturnType: 'ratio'` (the two + * are mutually exclusive in the editor); * - gauge/sum series project group-by dimensions as individual columns while * histogram series keep their single Array GROUP_ALIAS column, so grouped * histogram rows never join with grouped gauge/sum rows (each branch class @@ -2284,17 +2308,45 @@ async function renderMultiSeriesMetricChartConfig( throw new Error('multi-series metric charts require an array select'); } - // User-facing output column names in select order. Two series can resolve - // to the same alias (e.g. the same aggregation filtered vs unfiltered); - // suffix collisions with the split index so the operands stay distinct. + // User-facing output column names in select order, followed by formula + // column names in formulas order. Two columns can resolve to the same name + // (e.g. the same aggregation filtered vs unfiltered, or a formula aliased + // like an operand); suffix collisions with the column index so they stay + // distinct. + const formulas = chartConfig.formulas ?? []; const outputNames: string[] = []; + const formulaColumns: { name: string; ast: FormulaAst }[] = []; { const seen = new Set(); - select.forEach((s, splitIdx) => { - const base = s.alias ?? ''; - const name = seen.has(base) ? `${base}__${splitIdx}` : base; + const uniqueName = (base: string, columnIdx: number) => { + const name = seen.has(base) ? `${base}__${columnIdx}` : base; seen.add(name); - outputNames.push(name); + return name; + }; + select.forEach((s, splitIdx) => { + outputNames.push(uniqueName(s.alias ?? '', splitIdx)); + }); + formulas.forEach((f, formulaIdx) => { + // Parse + validate against the chart's series before rendering any + // SQL. Persisted configs should already be valid (the editor validates + // on save); this is a render-time guard so a stale or hand-built + // config fails with a structured message instead of a ClickHouse error. + const parsed = validateFormula(f.expression, { + seriesCount: select.length, + }); + if (!parsed.ok) { + throw new Error( + `Invalid formula "${f.expression}": ${parsed.errors + .map(e => e.message) + .join('; ')}`, + ); + } + formulaColumns.push({ + // A formula column is named by its alias, falling back to the raw + // expression text (mirrors DerivedColumnSchema.alias semantics). + name: uniqueName(f.alias || f.expression, select.length + formulaIdx), + ast: parsed.ast, + }); }); } @@ -2325,9 +2377,13 @@ async function renderMultiSeriesMetricChartConfig( // a non-final UNION branch). const branches = await Promise.all( select.map(async (s, splitIdx) => { + // Formulas belong to the composed outer projection only — a branch + // carrying them would recurse back into this function. const branchConfig: ChartConfigWithOptDateRangeEx = { ...chartConfig, select: [{ ...s, alias: MULTI_SERIES_VALUE_ALIAS }], + formulas: undefined, + showOperandSeries: undefined, }; const rendered = await renderChartConfig( branchConfig, @@ -2412,11 +2468,34 @@ async function renderMultiSeriesMetricChartConfig( const valueExprFor = (splitIdx: number) => `anyOrNullIf(\`${MULTI_SERIES_VALUE_ALIAS}\`, \`${MULTI_SERIES_IDX_ALIAS}\` = ${splitIdx})`; + // Formulas supersede the ratio toggle (mutually exclusive in the editor; + // rendering stays deterministic if a hand-built config carries both). + const hasFormulas = formulaColumns.length > 0; const isRatio = - chartConfig.seriesReturnType === 'ratio' && select.length === 2; + !hasFormulas && + chartConfig.seriesReturnType === 'ratio' && + select.length === 2; const projection: string[] = []; - if (isRatio) { + if (hasFormulas) { + // Value columns first, in select order, then the formula column(s) — + // the positional contract of useChartNumberFormats. showOperandSeries: + // false drops the raw operand columns from the projection (the union + // still computes every branch; the formula references them via the + // pivot expressions). + if (chartConfig.showOperandSeries !== false) { + outputNames.forEach((outputName, splitIdx) => { + projection.push( + `${valueExprFor(splitIdx)} AS ${quotedColumnName(outputName)}`, + ); + }); + } + formulaColumns.forEach(({ name, ast }) => { + projection.push( + `${compileFormulaAst(ast, valueExprFor)} AS ${quotedColumnName(name)}`, + ); + }); + } else if (isRatio) { // A group absent from the (filtered) numerator contributes zero, not // "no data" — so a zero-error group reads 0%, not N/A. A missing or zero // denominator makes the quotient NULL, which renders as a gap. @@ -2437,19 +2516,21 @@ async function renderMultiSeriesMetricChartConfig( // same-alias ratio reads "avg(x)/avg(x)", not "avg(x)/avg(x)__1". const ratioName = `${outputNames[0]}/${outputNames[1].replace(/__\d+$/, '')}`; projection.push( - `${numerator} / nullif(${denominator}, 0) AS "${ratioName}"`, + `${numerator} / nullif(${denominator}, 0) AS ${quotedColumnName(ratioName)}`, ); } else { outputNames.forEach((outputName, splitIdx) => { - projection.push(`${valueExprFor(splitIdx)} AS "${outputName}"`); + projection.push( + `${valueExprFor(splitIdx)} AS ${quotedColumnName(outputName)}`, + ); }); } // Pass the group and bucket columns through under their original names // (which are not knowable node-side for expression group-bys), and group // by exactly those columns. GROUP BY ALL expands to every non-aggregate - // SELECT expression, i.e. the * EXCEPT list; the pivot/ratio expressions - // contain aggregate functions and are excluded. With no passthrough + // SELECT expression, i.e. the * EXCEPT list; the pivot/ratio/formula + // expressions contain aggregate functions and are excluded. With no passthrough // columns at all (a number chart) the implicit global aggregation merges // everything into one row. const hasPassthroughColumns = @@ -2494,11 +2575,16 @@ export async function renderChartConfig( // A metric chart with multiple series composes one query per series into a // single UNION ALL + pivot statement (each metric type needs its own CTE // scaffolding, and different types read different physical tables). The - // per-series branches recurse through this function with a single select. + // per-series branches recurse through this function with a single select + // (and without formulas). A single-series chart with a formula (e.g. + // `A * 100`) also takes the composed path — the formula projects over the + // pivoted value columns, which only the composed shape produces. if ( isMetricChartConfig(substitutedChartConfig) && Array.isArray(substitutedChartConfig.select) && - substitutedChartConfig.select.length > 1 + (substitutedChartConfig.select.length > 1 || + (substitutedChartConfig.select.length === 1 && + hasMetricFormulas(substitutedChartConfig))) ) { return renderMultiSeriesMetricChartConfig( substitutedChartConfig,