From bbe84408b27a9b7cc9c381e01b52f54830d33f52 Mon Sep 17 00:00:00 2001 From: JivusAyrus Date: Fri, 17 Jul 2026 17:58:02 +0530 Subject: [PATCH 1/2] feat: add traces_detail_by_time table and materialized view for enhanced trace data retrieval --- .../20260707090829_traces_detail_by_time.sql | 48 +++++++++++++++++++ ...0260707090830_traces_detail_by_time_mv.sql | 44 +++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 controlplane/clickhouse/migrations/20260707090829_traces_detail_by_time.sql create mode 100644 controlplane/clickhouse/migrations/20260707090830_traces_detail_by_time_mv.sql diff --git a/controlplane/clickhouse/migrations/20260707090829_traces_detail_by_time.sql b/controlplane/clickhouse/migrations/20260707090829_traces_detail_by_time.sql new file mode 100644 index 0000000000..3ebf46e047 --- /dev/null +++ b/controlplane/clickhouse/migrations/20260707090829_traces_detail_by_time.sql @@ -0,0 +1,48 @@ +-- migrate:up + +-- + +CREATE TABLE IF NOT EXISTS traces_detail_by_time ( + TraceId String CODEC (ZSTD(3)), + SpanId String CODEC (ZSTD(3)), + Timestamp DateTime('UTC') CODEC (Delta(4), ZSTD(3)), + OperationName String CODEC (ZSTD(3)), + OperationType LowCardinality(String) CODEC (ZSTD(3)), + FederatedGraphID String CODEC(ZSTD(3)), + OrganizationID LowCardinality(String) CODEC(ZSTD(3)), + Duration Int64 CODEC(ZSTD(3)), + StatusCode LowCardinality(String) CODEC (ZSTD(3)), + HasError bool CODEC(ZSTD(3)), + StatusMessage String CODEC (ZSTD(3)), + OperationHash String CODEC (ZSTD(3)), + OperationContent String CODEC (ZSTD(3)), + OperationPersistedID String CODEC (ZSTD(3)), + HttpStatusCode String CODEC (ZSTD(3)), + HttpHost String CODEC (ZSTD(3)), + HttpUserAgent String CODEC (ZSTD(3)), + HttpMethod String CODEC (ZSTD(3)), + HttpTarget String CODEC (ZSTD(3)), + ClientName String CODEC (ZSTD(3)), + ClientVersion String CODEC (ZSTD(3)), + Subscription Bool CODEC(ZSTD(3)), + + -- Indexes for filtering; the table serves as the source for the raw traces view. + INDEX idx_operation_name OperationName TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_operation_type OperationType TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_operation_hash OperationHash TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_operation_persistent_id OperationPersistedID TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_client_name ClientName TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_client_version ClientVersion TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_http_status_code HttpStatusCode TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_duration Duration TYPE minmax GRANULARITY 1 +) ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +-- Scope-leading, native Timestamp: matches how the detail list queries the data. +ORDER BY ( + FederatedGraphID, OrganizationID, Timestamp, TraceId, SpanId +) +TTL toDateTime(Timestamp) + toIntervalDay(30) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; + +-- migrate:down + +DROP TABLE IF EXISTS cosmo.traces_detail_by_time diff --git a/controlplane/clickhouse/migrations/20260707090830_traces_detail_by_time_mv.sql b/controlplane/clickhouse/migrations/20260707090830_traces_detail_by_time_mv.sql new file mode 100644 index 0000000000..70802b686d --- /dev/null +++ b/controlplane/clickhouse/migrations/20260707090830_traces_detail_by_time_mv.sql @@ -0,0 +1,44 @@ +-- migrate:up + +-- Feeds `traces_detail_by_time` from otel_traces using the EXACT same transform and root-span +-- filter as `traces_mv`, so both read paths stay byte-for-byte equivalent during the migration +-- window. A TO-table MV's row order is governed by the target table's ORDER BY, so there is no +-- trailing ORDER BY here. No Timestamp cut-line: this MV ingests every root span from creation +-- forward; historical rows are backfilled separately (see +-- rfc/traces-analytics-performance/backfill_traces_detail_by_time.sh). + +CREATE MATERIALIZED VIEW IF NOT EXISTS cosmo.traces_detail_by_time_mv TO cosmo.traces_detail_by_time AS +SELECT + TraceId, + SpanId, + toDateTime(Timestamp, 'UTC') as Timestamp, + SpanAttributes [ 'wg.operation.name' ] as OperationName, + toLowCardinality(SpanAttributes [ 'wg.operation.type' ]) as OperationType, + SpanAttributes [ 'wg.federated_graph.id'] as FederatedGraphID, + toLowCardinality(SpanAttributes ['wg.organization.id']) as OrganizationID, + Duration, + toLowCardinality(StatusCode) as StatusCode, + if(StatusMessage == 'STATUS_CODE_ERROR' OR position(SpanAttributes['http.status_code'],'5') = 1 OR position(SpanAttributes['http.status_code'],'4') = 1 OR mapContains(SpanAttributes, 'wg.request.error'), true, false) as HasError, + StatusMessage, + SpanAttributes [ 'wg.operation.hash' ] as OperationHash, + SpanAttributes [ 'wg.operation.content' ] as OperationContent, + SpanAttributes [ 'wg.operation.persisted_id' ] as OperationPersistedID, + SpanAttributes [ 'http.status_code' ] as HttpStatusCode, + SpanAttributes [ 'http.host' ] as HttpHost, + SpanAttributes [ 'http.user_agent' ] as HttpUserAgent, + SpanAttributes [ 'http.method' ] as HttpMethod, + SpanAttributes [ 'http.target' ] as HttpTarget, + SpanAttributes [ 'wg.client.name' ] as ClientName, + SpanAttributes [ 'wg.client.version' ] as ClientVersion, + mapContains(SpanAttributes, 'wg.subscription') as Subscription +FROM + cosmo.otel_traces +WHERE + -- Only include router root spans + SpanAttributes [ 'wg.router.root_span' ] = 'true' OR + -- For backwards compatibility (router < 0.61.2) + SpanAttributes [ 'wg.component.name' ] = 'router-server'; + +-- migrate:down + +DROP VIEW IF EXISTS cosmo.traces_detail_by_time_mv From c8652ffec13075f54c36f22fbd83ba08e3987b00 Mon Sep 17 00:00:00 2001 From: JivusAyrus Date: Fri, 17 Jul 2026 17:58:22 +0530 Subject: [PATCH 2/2] fix: lint --- .../migrations/20260707090829_traces_detail_by_time.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/controlplane/clickhouse/migrations/20260707090829_traces_detail_by_time.sql b/controlplane/clickhouse/migrations/20260707090829_traces_detail_by_time.sql index 3ebf46e047..38fc3b6794 100644 --- a/controlplane/clickhouse/migrations/20260707090829_traces_detail_by_time.sql +++ b/controlplane/clickhouse/migrations/20260707090829_traces_detail_by_time.sql @@ -1,7 +1,5 @@ -- migrate:up --- - CREATE TABLE IF NOT EXISTS traces_detail_by_time ( TraceId String CODEC (ZSTD(3)), SpanId String CODEC (ZSTD(3)),