diff --git a/controlplane/clickhouse/migrations/20260707090829_traces_detail_by_time.sql b/controlplane/clickhouse/migrations/20260707090829_traces_detail_by_time.sql new file mode 100644 index 0000000000..38fc3b6794 --- /dev/null +++ b/controlplane/clickhouse/migrations/20260707090829_traces_detail_by_time.sql @@ -0,0 +1,46 @@ +-- migrate:up + +CREATE TABLE IF NOT EXISTS traces_detail_by_time ( + TraceId String CODEC (ZSTD(3)), + SpanId String CODEC (ZSTD(3)), + Timestamp DateTime('UTC') CODEC (Delta(4), ZSTD(3)), + OperationName String CODEC (ZSTD(3)), + OperationType LowCardinality(String) CODEC (ZSTD(3)), + FederatedGraphID String CODEC(ZSTD(3)), + OrganizationID LowCardinality(String) CODEC(ZSTD(3)), + Duration Int64 CODEC(ZSTD(3)), + StatusCode LowCardinality(String) CODEC (ZSTD(3)), + HasError bool CODEC(ZSTD(3)), + StatusMessage String CODEC (ZSTD(3)), + OperationHash String CODEC (ZSTD(3)), + OperationContent String CODEC (ZSTD(3)), + OperationPersistedID String CODEC (ZSTD(3)), + HttpStatusCode String CODEC (ZSTD(3)), + HttpHost String CODEC (ZSTD(3)), + HttpUserAgent String CODEC (ZSTD(3)), + HttpMethod String CODEC (ZSTD(3)), + HttpTarget String CODEC (ZSTD(3)), + ClientName String CODEC (ZSTD(3)), + ClientVersion String CODEC (ZSTD(3)), + Subscription Bool CODEC(ZSTD(3)), + + -- Indexes for filtering; the table serves as the source for the raw traces view. + INDEX idx_operation_name OperationName TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_operation_type OperationType TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_operation_hash OperationHash TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_operation_persistent_id OperationPersistedID TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_client_name ClientName TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_client_version ClientVersion TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_http_status_code HttpStatusCode TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_duration Duration TYPE minmax GRANULARITY 1 +) ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +-- Scope-leading, native Timestamp: matches how the detail list queries the data. +ORDER BY ( + FederatedGraphID, OrganizationID, Timestamp, TraceId, SpanId +) +TTL toDateTime(Timestamp) + toIntervalDay(30) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; + +-- migrate:down + +DROP TABLE IF EXISTS cosmo.traces_detail_by_time diff --git a/controlplane/clickhouse/migrations/20260707090830_traces_detail_by_time_mv.sql b/controlplane/clickhouse/migrations/20260707090830_traces_detail_by_time_mv.sql new file mode 100644 index 0000000000..70802b686d --- /dev/null +++ b/controlplane/clickhouse/migrations/20260707090830_traces_detail_by_time_mv.sql @@ -0,0 +1,44 @@ +-- migrate:up + +-- Feeds `traces_detail_by_time` from otel_traces using the EXACT same transform and root-span +-- filter as `traces_mv`, so both read paths stay byte-for-byte equivalent during the migration +-- window. A TO-table MV's row order is governed by the target table's ORDER BY, so there is no +-- trailing ORDER BY here. No Timestamp cut-line: this MV ingests every root span from creation +-- forward; historical rows are backfilled separately (see +-- rfc/traces-analytics-performance/backfill_traces_detail_by_time.sh). + +CREATE MATERIALIZED VIEW IF NOT EXISTS cosmo.traces_detail_by_time_mv TO cosmo.traces_detail_by_time AS +SELECT + TraceId, + SpanId, + toDateTime(Timestamp, 'UTC') as Timestamp, + SpanAttributes [ 'wg.operation.name' ] as OperationName, + toLowCardinality(SpanAttributes [ 'wg.operation.type' ]) as OperationType, + SpanAttributes [ 'wg.federated_graph.id'] as FederatedGraphID, + toLowCardinality(SpanAttributes ['wg.organization.id']) as OrganizationID, + Duration, + toLowCardinality(StatusCode) as StatusCode, + if(StatusMessage == 'STATUS_CODE_ERROR' OR position(SpanAttributes['http.status_code'],'5') = 1 OR position(SpanAttributes['http.status_code'],'4') = 1 OR mapContains(SpanAttributes, 'wg.request.error'), true, false) as HasError, + StatusMessage, + SpanAttributes [ 'wg.operation.hash' ] as OperationHash, + SpanAttributes [ 'wg.operation.content' ] as OperationContent, + SpanAttributes [ 'wg.operation.persisted_id' ] as OperationPersistedID, + SpanAttributes [ 'http.status_code' ] as HttpStatusCode, + SpanAttributes [ 'http.host' ] as HttpHost, + SpanAttributes [ 'http.user_agent' ] as HttpUserAgent, + SpanAttributes [ 'http.method' ] as HttpMethod, + SpanAttributes [ 'http.target' ] as HttpTarget, + SpanAttributes [ 'wg.client.name' ] as ClientName, + SpanAttributes [ 'wg.client.version' ] as ClientVersion, + mapContains(SpanAttributes, 'wg.subscription') as Subscription +FROM + cosmo.otel_traces +WHERE + -- Only include router root spans + SpanAttributes [ 'wg.router.root_span' ] = 'true' OR + -- For backwards compatibility (router < 0.61.2) + SpanAttributes [ 'wg.component.name' ] = 'router-server'; + +-- migrate:down + +DROP VIEW IF EXISTS cosmo.traces_detail_by_time_mv