diff --git a/source/common/stats/BUILD b/source/common/stats/BUILD index a6a40fb51108c..9b794ca862749 100644 --- a/source/common/stats/BUILD +++ b/source/common/stats/BUILD @@ -195,7 +195,10 @@ envoy_cc_library( envoy_cc_library( name = "symbol_table_lib", srcs = ["symbol_table.cc"], - hdrs = ["symbol_table.h"], + hdrs = [ + "symbol_table.h", + "well_known_tokens.h", + ], deps = [ ":recent_lookups_lib", "//source/common/common:assert_lib", diff --git a/source/common/stats/symbol_table.cc b/source/common/stats/symbol_table.cc index 28279ccdb9da4..dfb8add82de61 100644 --- a/source/common/stats/symbol_table.cc +++ b/source/common/stats/symbol_table.cc @@ -8,11 +8,17 @@ #include "source/common/common/assert.h" #include "source/common/common/logger.h" #include "source/common/common/utility.h" +#include "source/common/stats/well_known_tokens.h" #include "absl/strings/str_cat.h" namespace Envoy { namespace Stats { +namespace { + +// The well-known token table lives in well_known_tokens.h. WellKnownTokensNum is +// the number of reserved symbol slots (including the unused index-0 sentinel). +static constexpr Symbol WellKnownTokensNum = sizeof(kWellKnownTokens) / sizeof(kWellKnownTokens[0]); // When storing Symbol arrays, we disallow Symbol 0, which is the only Symbol // that will decode into uint8_t array starting (and ending) with {0}. Thus we @@ -21,9 +27,25 @@ namespace Stats { // used for dynamically discovered stat-name tokens where you don't want to take // a symbol table lock, and would rather pay extra memory overhead to store the // tokens as fully elaborated strings. -static constexpr Symbol FirstValidSymbol = 1; +static constexpr Symbol FirstValidSymbol = WellKnownTokensNum; static constexpr uint8_t LiteralStringIndicator = 0; +// Lock-free encode-side lookup: token string -> symbol ID. Built once at first +// access; safe for concurrent readers afterward. +const absl::flat_hash_map& wellKnownEncodeMap() { + static const auto* m = []() { + auto* m = new absl::flat_hash_map(); + m->reserve(WellKnownTokensNum); + for (Symbol i = 1; i < WellKnownTokensNum; ++i) { + (*m)[kWellKnownTokens[i]] = i; + } + return m; + }(); + return *m; +} + +} // namespace + #ifndef ENVOY_CONFIG_COVERAGE void StatName::debugPrint() { // TODO(jmarantz): capture this functionality (always prints regardless of @@ -166,6 +188,29 @@ class SymbolTable::Encoding::TokenIter { return TokenType::Symbol; } + // Quick scan of the remaining tokens to count how many there are without decoding them or + // output them. + size_t remainingTokens() const { + const uint8_t* array = array_; + size_t size = size_; + + size_t count = 0; + for (size_t i = 0; i < size;) { + if (array[i] == LiteralStringIndicator) { + // Skip the literal string indicator. + ++i; + std::pair length_consumed = decodeNumber(array + i); + i += length_consumed.second; // Skip the varint-encoded length. + i += length_consumed.first; // Skip the literal string data. + } else { + std::pair symbol_consumed = decodeNumber(array + i); + i += symbol_consumed.second; // Skip the varint-encoded symbol. + } + ++count; + } + return count; + } + /** @return the current string_view -- only valid to call if next()==TokenType::StringView */ absl::string_view stringView() const { #ifndef NDEBUG @@ -236,12 +281,35 @@ bool StatName::startsWith(StatName prefix) const { std::vector SymbolTable::decodeStrings(StatName stat_name) const { std::vector strings; - Thread::LockGuard lock(lock_); - Encoding::decodeTokens( - stat_name, - [this, &strings](Symbol symbol) - ABSL_NO_THREAD_SAFETY_ANALYSIS { strings.push_back(fromSymbol(symbol)); }, - [&strings](absl::string_view str) { strings.push_back(str); }); + absl::InlinedVector, 8> symbol_positions; + Encoding::TokenIter iter(stat_name); + strings.reserve(iter.remainingTokens()); + + for (Encoding::TokenIter::TokenType type = iter.next(); + type != Encoding::TokenIter::TokenType::End; type = iter.next()) { + if (type == Encoding::TokenIter::TokenType::Symbol) { + const Symbol symbol = iter.symbol(); + if (symbol < WellKnownTokensNum) { + strings.push_back(kWellKnownTokens[symbol]); + } else { + symbol_positions.emplace_back(symbol, strings.size()); + strings.push_back(absl::string_view()); // placeholder to be filled in later. + } + } else { + ASSERT(type == Encoding::TokenIter::TokenType::StringView); + strings.push_back(iter.stringView()); + } + } + + if (!symbol_positions.empty()) { + // Only take the lock if we have dynamic symbols to decode. We have already decoded the + // well-known symbols and the literal string symbols. + Thread::LockGuard lock(lock_); + for (const auto& symbol_position : symbol_positions) { + strings[symbol_position.second] = fromSymbol(symbol_position.first); + } + } + return strings; } @@ -285,19 +353,34 @@ void SymbolTable::addTokensToEncoding(const absl::string_view name, Encoding& en // string-splitting and prepare a temp vector of Symbol first. const std::vector tokens = absl::StrSplit(name, '.'); std::vector symbols; - symbols.reserve(tokens.size()); + symbols.resize(tokens.size()); + absl::InlinedVector, 8> un_well_known_token_positions; + const auto& well_known_encode_map = wellKnownEncodeMap(); + + for (size_t i = 0; i < tokens.size(); ++i) { + const absl::string_view token = tokens[i]; + const size_t hash = absl::Hash{}(token); + auto well_known_search = well_known_encode_map.find(token, hash); + if (well_known_search != well_known_encode_map.end()) { + symbols[i] = well_known_search->second; + } else { + un_well_known_token_positions.emplace_back(i, hash); + } + } // Now take the lock and populate the Symbol objects, which involves bumping // ref-counts in this. { Thread::LockGuard lock(lock_); recent_lookups_.lookup(name); - for (auto& token : tokens) { + for (const auto& token_position : un_well_known_token_positions) { + const size_t i = token_position.first; + const size_t hash = token_position.second; // TODO(jmarantz): consider using StatNameDynamicStorage for tokens with // length below some threshold, say 4 bytes. It might be preferable not to // reserve Symbols for every 3 digit number found (for example) in ipv4 // addresses. - symbols.push_back(toSymbol(token)); + symbols[i] = toUnWellKnownSymbol(tokens[i], hash); } } @@ -319,8 +402,24 @@ void SymbolTable::incRefCount(const StatName& stat_name) { // Before taking the lock, decode the array of symbols from the SymbolTable::Storage. const SymbolVec symbols = Encoding::decodeSymbols(stat_name); + // Well-known symbols are statically allocated and have no ref count. If the + // StatName only references well-known symbols, skip the lock entirely. + bool any_dynamic = false; + for (Symbol s : symbols) { + if (s >= FirstValidSymbol) { + any_dynamic = true; + break; + } + } + if (!any_dynamic) { + return; + } + Thread::LockGuard lock(lock_); for (Symbol symbol : symbols) { + if (symbol < FirstValidSymbol) { + continue; + } auto decode_search = decode_map_.find(symbol); ASSERT(decode_search != decode_map_.end(), @@ -341,8 +440,24 @@ void SymbolTable::free(const StatName& stat_name) { // Before taking the lock, decode the array of symbols from the SymbolTable::Storage. const SymbolVec symbols = Encoding::decodeSymbols(stat_name); + // Well-known symbols are statically allocated and have no ref count. If the + // StatName only references well-known symbols, skip the lock entirely. + bool any_dynamic = false; + for (Symbol s : symbols) { + if (s >= FirstValidSymbol) { + any_dynamic = true; + break; + } + } + if (!any_dynamic) { + return; + } + Thread::LockGuard lock(lock_); for (Symbol symbol : symbols) { + if (symbol < FirstValidSymbol) { + continue; + } auto decode_search = decode_map_.find(symbol); ASSERT(decode_search != decode_map_.end()); @@ -439,9 +554,9 @@ StatNameSetPtr SymbolTable::makeSet(absl::string_view name) { return stat_name_set; } -Symbol SymbolTable::toSymbol(absl::string_view sv) { +Symbol SymbolTable::toUnWellKnownSymbol(absl::string_view sv, size_t hash) { Symbol result; - auto encode_find = encode_map_.find(sv); + auto encode_find = encode_map_.find(sv, hash); // If the string segment doesn't already exist, if (encode_find == encode_map_.end()) { // We create the actual string, place it in the decode_map_, and then insert @@ -467,6 +582,12 @@ Symbol SymbolTable::toSymbol(absl::string_view sv) { absl::string_view SymbolTable::fromSymbol(const Symbol symbol) const ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) { + if (symbol < WellKnownTokensNum) { + // Well-known symbols are resolved without consulting decode_map_. The + // caller still holds the lock per the contract, but this skips the hash + // lookup entirely. + return kWellKnownTokens[symbol]; + } auto search = decode_map_.find(symbol); RELEASE_ASSERT(search != decode_map_.end(), "no such symbol"); return search->second->toStringView(); diff --git a/source/common/stats/symbol_table.h b/source/common/stats/symbol_table.h index 8f2fc96631dd8..74f4c6508194f 100644 --- a/source/common/stats/symbol_table.h +++ b/source/common/stats/symbol_table.h @@ -446,10 +446,13 @@ class SymbolTable final { /** * Convenience function for encode(), symbolizing one string segment at a time. * - * @param sv the individual string to be encoded as a symbol. + * @param sv the individual string to be encoded as a symbol. This should not contains the + * well-known tokens. + * @param hash the precomputed hash of the string. * @return Symbol the encoded string. */ - Symbol toSymbol(absl::string_view sv) ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_); + Symbol toUnWellKnownSymbol(absl::string_view sv, size_t hash) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_); /** * Convenience function for decode(), decoding one symbol at a time. diff --git a/source/common/stats/well_known_tokens.h b/source/common/stats/well_known_tokens.h new file mode 100644 index 0000000000000..8f79aa8a73667 --- /dev/null +++ b/source/common/stats/well_known_tokens.h @@ -0,0 +1,619 @@ +#pragma once + +#include "absl/strings/string_view.h" + +namespace Envoy { +namespace Stats { + +// Statically-assigned, lock-free symbol IDs for common stat tokens. Symbols +// 1..WellKnownTokensNum-1 are reserved for this table; dynamic tokens start at +// FirstValidSymbol. +// +// Properties relied on by the rest of SymbolTable: +// * Index 0 is unused (matches LiteralStringIndicator). +// * Indices 1..WellKnownTokensNum-1 are the symbol IDs; the slot at that index +// is the string for that symbol. +// * These entries are never inserted into encode_map_/decode_map_ and have +// no ref count. +// +// The array itself costs almost nothing (it only holds string_view pointers into +// read-only binary data), so growing this list is cheap. The one cost is that a +// symbol > 127 takes 2 varint bytes instead of 1 in every StatName that +// references it. Therefore the *hottest* tokens (scope prefixes, per-connection +// and per-request counters, common response codes) are kept in the first 127 +// slots, and the lower-frequency long tail is appended afterwards. Symbol IDs are +// derived from the array index and never persisted across processes, so this list +// may be freely reordered or extended between builds. +// +// Sourced from the canonical COUNTER/GAUGE/HISTOGRAM/TEXT_READOUT macros across +// Envoy's core subsystems, the per-response-code stats generated by CodeStatsImpl, +// and the well-known tag names. +inline constexpr absl::string_view kWellKnownTokens[] = { + "", // index 0 - unused sentinel (LiteralStringIndicator). + // Top-level scope prefixes. + "cluster", + "http", + "listener", + "server", + "runtime", + "tcp", + "udp", + "vhost", + "vcluster", + "route", + // Kubernetes / Istio cluster-name tokens. Cluster names are split on '.', so + // service FQDNs like "reviews.default.svc.cluster.local" and Istio + // "outbound|9080||..." names emit these tokens on nearly every cluster stat + // in a mesh deployment. "cluster" is already covered above. "default" doubles + // as the circuit-breaker default-priority token. + "svc", + "local", + "default", + "outbound", + "inbound", + "kube-system", + "istio-system", + // Common tag-name tokens. The "envoy" prefix is shared by almost every tag + // name. + "envoy", + "response_code", + "response_code_class", + "cluster_name", + "http_conn_manager_prefix", + "listener_address", + "worker_id", + "virtual_host", + "virtual_cluster", + // HTTP downstream connection counters/gauges. + "downstream_cx_total", + "downstream_cx_active", + "downstream_cx_destroy", + "downstream_cx_destroy_local", + "downstream_cx_destroy_remote", + "downstream_cx_rx_bytes_total", + "downstream_cx_rx_bytes_buffered", + "downstream_cx_tx_bytes_total", + "downstream_cx_tx_bytes_buffered", + "downstream_cx_http1_total", + "downstream_cx_http2_total", + "downstream_cx_http3_total", + "downstream_cx_http1_active", + "downstream_cx_http2_active", + "downstream_cx_length_ms", + "downstream_cx_protocol_error", + // HTTP downstream request counters/gauges. + "downstream_rq_total", + "downstream_rq_active", + "downstream_rq_completed", + "downstream_rq_1xx", + "downstream_rq_2xx", + "downstream_rq_3xx", + "downstream_rq_4xx", + "downstream_rq_5xx", + "downstream_rq_time", + "downstream_rq_timeout", + "downstream_rq_rx_reset", + "downstream_rq_tx_reset", + "downstream_rq_http1_total", + "downstream_rq_http2_total", + // Listener counters/gauges. + "no_filter_chain_match", + "downstream_pre_cx_active", + // Upstream cluster connection counters/gauges. + "upstream_cx_total", + "upstream_cx_active", + "upstream_cx_connect_fail", + "upstream_cx_connect_timeout", + "upstream_cx_connect_ms", + "upstream_cx_length_ms", + "upstream_cx_destroy", + "upstream_cx_destroy_local", + "upstream_cx_destroy_remote", + "upstream_cx_destroy_with_active_rq", + "upstream_cx_rx_bytes_total", + "upstream_cx_rx_bytes_buffered", + "upstream_cx_tx_bytes_total", + "upstream_cx_tx_bytes_buffered", + "upstream_cx_overflow", + "upstream_cx_none_healthy", + "upstream_cx_protocol_error", + "upstream_cx_http1_total", + "upstream_cx_http2_total", + "upstream_cx_http3_total", + // Upstream cluster request counters/gauges. + "upstream_rq_total", + "upstream_rq_active", + "upstream_rq_completed", + "upstream_rq_pending_total", + "upstream_rq_pending_active", + "upstream_rq_pending_overflow", + "upstream_rq_pending_failure_eject", + "upstream_rq_cancelled", + "upstream_rq_maintenance_mode", + "upstream_rq_1xx", + "upstream_rq_2xx", + "upstream_rq_3xx", + "upstream_rq_4xx", + "upstream_rq_5xx", + "upstream_rq_time", + "upstream_rq_timeout", + "upstream_rq_per_try_timeout", + "upstream_rq_rx_reset", + "upstream_rq_tx_reset", + "upstream_rq_retry", + "upstream_rq_retry_overflow", + "upstream_rq_retry_success", + // Per-response-code upstream request counters (the token format is + // upstream_rq_). + "upstream_rq_200", + "upstream_rq_201", + "upstream_rq_204", + "upstream_rq_206", + "upstream_rq_301", + "upstream_rq_302", + "upstream_rq_304", + "upstream_rq_400", + "upstream_rq_401", + "upstream_rq_403", + "upstream_rq_404", + "upstream_rq_429", + "upstream_rq_500", + "upstream_rq_502", + "upstream_rq_503", + "upstream_rq_504", + // Cluster load balancing and membership. + "bind_errors", + "lb_healthy_panic", + "membership_change", + "membership_healthy", + "membership_total", + "membership_excluded", + "membership_degraded", + "health_check", + + // ---------------------------------------------------------------------- + // Lower-frequency long tail. Tokens below may encode to 2 varint bytes + // (symbol > 127); this is fine since the stats here are far less hot. + // ---------------------------------------------------------------------- + + // Additional per-response-code upstream request counters (upstream_rq_). + "upstream_rq_100", + "upstream_rq_101", + "upstream_rq_202", + "upstream_rq_203", + "upstream_rq_205", + "upstream_rq_300", + "upstream_rq_303", + "upstream_rq_305", + "upstream_rq_307", + "upstream_rq_308", + "upstream_rq_402", + "upstream_rq_405", + "upstream_rq_406", + "upstream_rq_407", + "upstream_rq_408", + "upstream_rq_409", + "upstream_rq_410", + "upstream_rq_411", + "upstream_rq_412", + "upstream_rq_413", + "upstream_rq_414", + "upstream_rq_415", + "upstream_rq_416", + "upstream_rq_417", + "upstream_rq_421", + "upstream_rq_422", + "upstream_rq_426", + "upstream_rq_428", + "upstream_rq_431", + "upstream_rq_451", + "upstream_rq_501", + "upstream_rq_505", + "upstream_rq_507", + "upstream_rq_511", + "upstream_rq_unknown", + + // Additional HTTP connection manager downstream stats. + "downstream_cx_delayed_close_timeout", + "downstream_cx_destroy_active_rq", + "downstream_cx_destroy_local_active_rq", + "downstream_cx_destroy_remote_active_rq", + "downstream_cx_drain_close", + "downstream_cx_idle_timeout", + "downstream_cx_max_duration_reached", + "downstream_cx_max_requests_reached", + "downstream_cx_overload_disable_keepalive", + "downstream_cx_ssl_total", + "downstream_cx_ssl_active", + "downstream_cx_upgrades_total", + "downstream_cx_upgrades_active", + "downstream_cx_http3_active", + "downstream_cx_http1_soft_drain", + "downstream_flow_control_paused_reading_total", + "downstream_flow_control_resumed_reading_total", + "downstream_rq_http3_total", + "downstream_rq_failed_path_normalization", + "downstream_rq_idle_timeout", + "downstream_rq_header_timeout", + "downstream_rq_non_relative_path", + "downstream_rq_overload_close", + "downstream_rq_redirected_with_normalized_path", + "downstream_rq_rejected_via_ip_detection", + "downstream_rq_response_before_rq_complete", + "downstream_rq_too_many_premature_resets", + "downstream_rq_too_large", + "downstream_rq_max_duration_reached", + "downstream_rq_ws_on_non_ws_route", + "rs_too_large", + + // Additional listener stats. + "downstream_cx_overflow", + "downstream_cx_overload_reject", + "downstream_cx_transport_socket_connect_timeout", + "downstream_global_cx_overflow", + "downstream_pre_cx_timeout", + "downstream_listener_filter_remote_close", + "downstream_listener_filter_error", + "connections_accepted_per_socket_event", + "downstream_rx_datagram_dropped", + + // Additional cluster traffic stats. + "original_dst_host_invalid", + "retry_or_shadow_abandoned", + "upstream_cx_close_notify", + "upstream_cx_connect_attempts_exceeded", + "upstream_cx_connect_with_0_rtt", + "upstream_cx_destroy_local_with_active_rq", + "upstream_cx_destroy_remote_with_active_rq", + "upstream_cx_idle_timeout", + "upstream_cx_max_duration_reached", + "upstream_cx_max_requests", + "upstream_cx_pool_overflow", + "upstream_flow_control_backed_up_total", + "upstream_flow_control_drained_total", + "upstream_flow_control_paused_reading_total", + "upstream_flow_control_resumed_reading_total", + "upstream_internal_redirect_failed_total", + "upstream_internal_redirect_succeeded_total", + "upstream_rq_max_duration_reached", + "upstream_rq_active_overflow", + "upstream_rq_0rtt", + "upstream_rq_per_try_idle_timeout", + "upstream_rq_retry_backoff_exponential", + "upstream_rq_retry_backoff_ratelimited", + "upstream_rq_retry_limit_exceeded", + "upstream_rq_rx_reset_no_error", + "upstream_http3_broken", + "upstream_rq_per_cx", + "upstream_rq_headers_size", + "upstream_rq_headers_count", + "upstream_rq_body_size", + "upstream_rs_headers_size", + "upstream_rs_headers_count", + "upstream_rs_body_size", + "upstream_rq_timeout_budget_percent_used", + "upstream_rq_dropped", + "upstream_rq_drop_overload", + "max_host_weight", + + // Cluster load balancer stats. + "lb_local_cluster_not_ok", + "lb_recalculate_zone_structures", + "lb_subsets_created", + "lb_subsets_fallback", + "lb_subsets_fallback_panic", + "lb_subsets_removed", + "lb_subsets_selected", + "lb_subsets_active", + "lb_zone_cluster_too_small", + "lb_zone_no_capacity_left", + "lb_zone_routing_all_directly", + "lb_zone_routing_cross_zone", + "lb_zone_routing_sampled", + + // Cluster circuit breaker stats. + "circuit_breakers", + "high", + "cx_open", + "cx_pool_open", + "rq_open", + "rq_pending_open", + "rq_retry_open", + "remaining_cx", + "remaining_cx_pools", + "remaining_pending", + "remaining_retries", + "remaining_rq", + + // Cluster config-update stats. + "assignment_stale", + "assignment_timeout_received", + "assignment_use_cached", + "update_attempt", + "update_empty", + "update_failure", + "update_no_rebuild", + "update_success", + "warming_state", + + // Cluster manager stats. + "cluster_added", + "cluster_modified", + "cluster_removed", + "cluster_updated", + "cluster_updated_via_merge", + "update_merge_cancelled", + "update_out_of_merge_window", + "active_clusters", + "warming_clusters", + "clusters_inflated", + + // Listener manager stats. + "listener_added", + "listener_create_failure", + "listener_create_success", + "listener_in_place_updated", + "listener_modified", + "listener_removed", + "listener_stopped", + "total_filter_chains_draining", + "total_listeners_active", + "total_listeners_draining", + "total_listeners_warming", + "workers_started", + + // Router / route / virtual-cluster stats. + "no_cluster", + "no_route", + "passthrough_internal_redirect_bad_location", + "passthrough_internal_redirect_no_route", + "passthrough_internal_redirect_predicate", + "passthrough_internal_redirect_too_many_redirects", + "passthrough_internal_redirect_unsafe_scheme", + "rq_direct_response", + "rq_overload_local_reply", + "rq_redirect", + "rq_reset_after_downstream_response_started", + "rq_total", + "retry", + "other", + + // Outlier detection stats. + "ejections_active", + "ejections_overflow", + "ejections_total", + "ejections_enforced_total", + "ejections_success_rate", + "ejections_consecutive_5xx", + "ejections_detected_consecutive_5xx", + "ejections_enforced_consecutive_5xx", + "ejections_detected_consecutive_gateway_failure", + "ejections_enforced_consecutive_gateway_failure", + "ejections_detected_success_rate", + "ejections_enforced_success_rate", + "ejections_detected_failure_percentage", + "ejections_enforced_failure_percentage", + "ejections_detected_consecutive_local_origin_failure", + "ejections_enforced_consecutive_local_origin_failure", + "ejections_detected_local_origin_success_rate", + "ejections_enforced_local_origin_success_rate", + "ejections_detected_local_origin_failure_percentage", + "ejections_enforced_local_origin_failure_percentage", + "ejections_detected_degradation", + + // Health checker stats. + "attempt", + "failure", + "network_failure", + "passive_failure", + "success", + "verify_cluster", + "degraded", + "healthy", + + // Per-host stats. + "cx_total", + "cx_active", + "cx_connect_fail", + "rq_error", + "rq_success", + "rq_timeout", + "rq_active", + "total_match_count", + + // TLS/SSL stats. + "connection_error", + "handshake", + "session_reused", + "no_certificate", + "fail_verify_no_cert", + "fail_verify_error", + "fail_verify_san", + "fail_verify_cert_hash", + "ocsp_staple_failed", + "ocsp_staple_omitted", + "ocsp_staple_responses", + "ocsp_staple_requests", + "was_key_usage_invalid", + "ssl_context_update_by_sds", + "upstream_context_secrets_not_ready", + "downstream_context_secrets_not_ready", + + // HTTP codec stats. + "dropped_headers_with_underscores", + "requests_rejected_with_underscores_in_headers", + "metadata_not_supported_error", + "response_flood", + "goaway_sent", + "header_overflow", + "headers_cb_no_stream", + "inbound_empty_frames_flood", + "inbound_priority_frames_flood", + "inbound_window_update_frames_flood", + "keepalive_timeout", + "metadata_empty_frames", + "outbound_control_flood", + "outbound_flood", + "rx_messaging_error", + "rx_reset", + "tx_reset", + "stream_refused_errors", + "trailers", + "tx_flush_timeout", + "streams_active", + "pending_send_bytes", + "quic_version_h3_29", + "quic_version_rfc_v1", + + // Runtime stats. + "deprecated_feature_use", + "deprecated_feature_seen_since_process_start", + "load_error", + "load_success", + "override_dir_exists", + "override_dir_not_exists", + "admin_overrides_active", + "num_keys", + "num_layers", + + // xDS subscription / control-plane stats. + "config_reload", + "config_reload_time_ms", + "init_fetch_timeout", + "update_rejected", + "update_time", + "update_duration", + "version_text", + "rate_limit_enforced", + "connected_state", + "pending_requests", + "identifier", + "key_rotation_failed", + "all_scopes", + "on_demand_scopes", + "active_scopes", + "config_fail", + "config_conflict", + "extension_config_missing", + "network_extension_config_missing", + + // Server stats. + "uptime", + "live", + "state", + "concurrency", + "version", + "memory_allocated", + "memory_heap_size", + "memory_physical_size", + "total_connections", + "parent_connections", + "hot_restart_epoch", + "stats_recent_lookups", + "initialization_time_ms", + "days_until_first_cert_expiring", + "seconds_until_first_ocsp_response_expiring", + "dropped_stat_flushes", + "debug_assertion_failures", + "envoy_bug_failures", + "envoy_notifications", + "dynamic_unknown_fields", + "static_unknown_fields", + "wip_protos", + "fips_mode", + + // Dispatcher stats. + "loop_duration_us", + "poll_delay_us", + + // TCP proxy stats. + "downstream_cx_no_route", + "early_data_received_count_total", + "idle_timeout", + "max_downstream_connection_duration", + "upstream_flush_total", + "upstream_flush_active", + "route_delayed_total", + + // UDP proxy stats. + "downstream_sess_total", + "downstream_sess_active", + "downstream_sess_no_route", + "downstream_sess_rx_bytes", + "downstream_sess_rx_datagrams", + "downstream_sess_rx_errors", + "downstream_sess_tx_bytes", + "downstream_sess_tx_datagrams", + "downstream_sess_tx_errors", + "session_filter_config_missing", + "sess_rx_datagrams", + "sess_rx_datagrams_dropped", + "sess_rx_errors", + "sess_tx_datagrams", + "sess_tx_errors", + "sess_tunnel_success", + "sess_tunnel_failure", + "sess_tunnel_buffer_overflow", + + // DNS cache stats. + "cache_load", + "dns_query_attempt", + "dns_query_success", + "dns_query_failure", + "dns_query_timeout", + "dns_rq_pending_overflow", + "host_added", + "host_removed", + "host_overflow", + "host_address_changed", + + // Health discovery / load reporter stats. + "requests", + "responses", + "errors", + "updates", + "retries", + + // Common tag-name suffix tokens. + "rds_route_config", + "scoped_rds_config", + "http_user_agent", + "ssl_cipher", + "ssl_curve", + "ssl_sigalg", + "ssl_version", + "cipher_suite", + "tls_certificate", + "xds_resource_name", + "proxy_protocol_version", + "proxy_protocol_prefix", + "google_grpc_client_prefix", + "clientssl_prefix", + "mongo_prefix", + "mongo_cmd", + "mongo_collection", + "mongo_callsite", + "ratelimit_prefix", + "local_http_ratelimit_prefix", + "local_network_ratelimit_prefix", + "local_listener_ratelimit_prefix", + "dns_filter_prefix", + "connection_limit_prefix", + "rbac_prefix", + "rbac_http_prefix", + "rbac_policy_name", + "tcp_prefix", + "udp_prefix", + "thrift_prefix", + "redis_prefix", + "ext_authz_prefix", + "fault_downstream_cluster", + "dynamo_operation", + "dynamo_table", + "dynamo_partition_id", + "grpc_bridge_service", + "grpc_bridge_method", +}; + +} // namespace Stats +} // namespace Envoy diff --git a/test/common/stats/symbol_table_impl_test.cc b/test/common/stats/symbol_table_impl_test.cc index 803b252e290e8..eeb0b4fa88a2f 100644 --- a/test/common/stats/symbol_table_impl_test.cc +++ b/test/common/stats/symbol_table_impl_test.cc @@ -242,7 +242,10 @@ class StatNameDeathTest : public StatNameTest { TEST_F(StatNameDeathTest, TestBadDecodes) { { // If a symbol doesn't exist, decoding it should trigger an ASSERT() and crash. - SymbolVec bad_symbol_vec = {1}; // symbol 0 is the empty symbol. + // Symbols below FirstValidSymbol are reserved for the well-known token table and + // always decode successfully, so we use monotonicCounter() -- the next symbol that + // would be allocated. It is past the well-known range and not yet in the table. + SymbolVec bad_symbol_vec = {monotonicCounter()}; EXPECT_DEATH(decodeSymbolVec(bad_symbol_vec), ""); } @@ -302,6 +305,12 @@ TEST_F(StatNameTest, FreePoolTest) { // a) the size of the table has not increased, and // b) the monotonically increasing counter has not risen to more than the maximum number of // coexisting symbols during the life of the table. + // + // The tokens used here ("1a", "2a", ...) are deliberately not well-known, so they are allocated + // dynamic symbols. Dynamic symbols start at FirstValidSymbol (past the reserved well-known + // range), so we assert the counter relative to its initial value rather than against absolute + // symbol IDs. + const Symbol base = monotonicCounter(); { makeStat("1a"); @@ -309,11 +318,11 @@ TEST_F(StatNameTest, FreePoolTest) { makeStat("3a"); makeStat("4a"); makeStat("5a"); - EXPECT_EQ(monotonicCounter(), 6); + EXPECT_EQ(monotonicCounter(), base + 5); EXPECT_EQ(table_.numSymbols(), 5); clearStorage(); } - EXPECT_EQ(monotonicCounter(), 6); + EXPECT_EQ(monotonicCounter(), base + 5); EXPECT_EQ(table_.numSymbols(), 0); // These are different strings being encoded, but they should recycle through the same symbols as @@ -323,11 +332,11 @@ TEST_F(StatNameTest, FreePoolTest) { makeStat("3b"); makeStat("4b"); makeStat("5b"); - EXPECT_EQ(monotonicCounter(), 6); + EXPECT_EQ(monotonicCounter(), base + 5); EXPECT_EQ(table_.numSymbols(), 5); makeStat("6"); - EXPECT_EQ(monotonicCounter(), 7); + EXPECT_EQ(monotonicCounter(), base + 6); EXPECT_EQ(table_.numSymbols(), 6); } @@ -780,58 +789,59 @@ TEST_F(StatNameTest, StatNameEqualityFastPaths) { } TEST_F(StatNameTest, EqualityWithMultiByteVarint) { - // Sweep segment counts around the varint boundary to catch off-by-one - // errors in decodeNumber's fast-path vs slow-path. Each segment gets a - // unique symbol assigned sequentially. Symbols 1-127 encode as 1 byte - // each; symbol 128+ encodes as 2 bytes. Pre-allocating throwaway symbols - // shifts the numbering so that different padding values produce different - // dataSizes for the same segment count. We verify that the sweep covers - // both sides of the 128 boundary (the fast-path/slow-path transition in - // the length-prefix varint). - bool seen[150] = {}; - - // Sweep a few potential padding numbers just to make sure we cover. This - // is over-testing but that's ok. + // Exercises decodeNumber's fast-path (single-byte) vs slow-path (multi-byte) for + // symbol varints, and confirms StatName equality/hashing/comparison are correct + // when both encodings appear in a name. + // + // With the two-level symbol table, dynamically-created tokens are assigned + // symbols starting at FirstValidSymbol (past the reserved well-known range), so + // they always encode as multi-byte varints (symbol >= SpilloverMask). To also + // exercise the single-byte fast path, each name is prefixed with the well-known + // "cluster" token, whose symbol is small (< SpilloverMask) and thus one byte. + // + // We sweep segment counts around the varint boundary and assert that both decode + // paths were actually taken. + bool seen_single_byte = false; + bool seen_multi_byte = false; + constexpr int lower_bound = static_cast(SymbolTable::Encoding::SpilloverMask) - 5; constexpr int upper_bound = static_cast(SymbolTable::Encoding::SpilloverMask) + 5; - for (int padding = 0; padding <= 10; ++padding) { - for (int num_segments = lower_bound; num_segments <= upper_bound; ++num_segments) { - clearStorage(); + for (int num_segments = lower_bound; num_segments <= upper_bound; ++num_segments) { + clearStorage(); - // Pre-allocate throwaway symbols to shift symbol numbering. - for (int p = 0; p < padding; ++p) { - makeStat(absl::StrCat("pad", p)); - } + // "cluster" is well-known (single-byte symbol); "s1".."s{n-1}" are dynamic + // (multi-byte symbols). + std::string long_name = "cluster"; + for (int i = 1; i < num_segments; ++i) { + absl::StrAppend(&long_name, ".s", i); + } - std::string long_name = "s0"; - for (int i = 1; i < num_segments; ++i) { - absl::StrAppend(&long_name, ".s", i); + // Two pool entries for the same name: distinct backing storage, identical content. + StatName x = makeStat(long_name); + StatName y = makeStat(long_name); + EXPECT_NE(x.dataIncludingSize(), y.dataIncludingSize()) << "segments=" << num_segments; + EXPECT_EQ(x, y) << "segments=" << num_segments; + EXPECT_EQ(x.hash(), y.hash()) << "segments=" << num_segments; + + for (Symbol symbol : getSymbols(x)) { + if (symbol < SymbolTable::Encoding::SpilloverMask) { + seen_single_byte = true; + } else { + seen_multi_byte = true; } + } - // Two pool entries for the same name: distinct backing storage, identical content. - StatName x = makeStat(long_name); - StatName y = makeStat(long_name); - size_t ds = x.dataSize(); - ASSERT_LT(ds, 150); - seen[ds] = true; - EXPECT_NE(x.dataIncludingSize(), y.dataIncludingSize()) - << "padding=" << padding << " segments=" << num_segments; - EXPECT_EQ(x, y) << "padding=" << padding << " segments=" << num_segments; - EXPECT_EQ(x.hash(), y.hash()) << "padding=" << padding << " segments=" << num_segments; - - // Same segment count but shifted names — memcmp mismatch. - std::string long_name_alt = "s1"; - for (int i = 2; i < num_segments + 1; ++i) { - absl::StrAppend(&long_name_alt, ".s", i); - } - StatName z = makeStat(long_name_alt); - EXPECT_NE(x, z) << "padding=" << padding << " segments=" << num_segments; + // Same segment count but shifted names — memcmp mismatch. + std::string long_name_alt = "cluster"; + for (int i = 2; i < num_segments + 1; ++i) { + absl::StrAppend(&long_name_alt, ".s", i); } + StatName z = makeStat(long_name_alt); + EXPECT_NE(x, z) << "segments=" << num_segments; } - // Confirm every dataSize across the varint boundary was exercised. - for (int i = lower_bound; i <= upper_bound; ++i) { - EXPECT_TRUE(seen[i]) << "dataSize " << i << " was never hit"; - } + // Confirm both the single-byte and multi-byte varint decode paths were exercised. + EXPECT_TRUE(seen_single_byte); + EXPECT_TRUE(seen_multi_byte); } TEST_F(StatNameTest, EqualitySizeMismatch) { diff --git a/test/common/stats/symbol_table_speed_test.cc b/test/common/stats/symbol_table_speed_test.cc index acda8a35a9550..30062e39759a6 100644 --- a/test/common/stats/symbol_table_speed_test.cc +++ b/test/common/stats/symbol_table_speed_test.cc @@ -15,6 +15,7 @@ #include "test/common/stats/make_elements_helper.h" #include "test/test_common/utility.h" +#include "absl/strings/str_cat.h" #include "absl/synchronization/blocking_counter.h" #include "benchmark/benchmark.h" @@ -288,6 +289,113 @@ static void bmSetStrings(benchmark::State& state) { } BENCHMARK(bmSetStrings); +// --------------------------------------------------------------------------- +// Kubernetes / Istio cluster stats serialization +// +// Models the stat names Envoy emits per upstream cluster in a service-mesh +// deployment. Istio names clusters +// "outbound|||..svc.cluster.local", so the per-cluster +// stat "cluster.." splits on '.' into mostly well-known +// tokens -- "cluster", the namespace (often "default"), "svc", "cluster", +// "local", and the stat suffix -- plus a single dynamic +// "outbound|||" token. Serializing these names back to strings +// (SymbolTable::toString) is the hot path on every stats flush (admin /stats, +// Prometheus, stats sinks), and it is exactly what the well-known token table is +// meant to accelerate: well-known tokens decode straight from the static array +// without taking the symbol-table lock. +// --------------------------------------------------------------------------- + +static std::vector +prepareK8sClusterStatNames(Envoy::Stats::StatNamePool& pool, uint32_t num_services) { + // A representative slice of ALL_CLUSTER_STATS plus per-response-code counters. + // Every suffix here is a well-known token. + const std::vector stat_suffixes = { + "upstream_cx_total", + "upstream_cx_active", + "upstream_cx_connect_fail", + "upstream_cx_connect_timeout", + "upstream_cx_rx_bytes_total", + "upstream_cx_tx_bytes_total", + "upstream_cx_destroy", + "upstream_cx_destroy_local", + "upstream_cx_destroy_remote", + "upstream_rq_total", + "upstream_rq_active", + "upstream_rq_pending_total", + "upstream_rq_pending_active", + "upstream_rq_timeout", + "upstream_rq_retry", + "upstream_rq_2xx", + "upstream_rq_4xx", + "upstream_rq_5xx", + "upstream_rq_200", + "upstream_rq_404", + "upstream_rq_503", + "membership_healthy", + "membership_total", + }; + // "default" is well-known; the others are dynamic, like most real namespaces. + const std::vector namespaces = {"default", "kube-system", "production", + "staging"}; + // A few realistic service names; uniqueness comes from the appended index so + // that each cluster gets a distinct dynamic token. + const std::vector services = {"reviews", "ratings", "productpage", "details"}; + + std::vector names; + names.reserve(static_cast(num_services) * stat_suffixes.size()); + for (uint32_t i = 0; i < num_services; ++i) { + const absl::string_view ns = namespaces[i % namespaces.size()]; + const absl::string_view svc = services[i % services.size()]; + // The single dynamic token is "outbound|9080||-". + const std::string cluster_scope = + absl::StrCat("cluster.outbound|9080||", svc, "-", i, ".", ns, ".svc.cluster.local"); + for (absl::string_view suffix : stat_suffixes) { + names.push_back(pool.add(absl::StrCat(cluster_scope, ".", suffix))); + } + } + return names; +} + +// Serialize (decode to string) the stats of a fleet of mesh clusters. +// NOLINTNEXTLINE(readability-identifier-naming) +static void bmK8sClusterStatsToString(benchmark::State& state) { + Envoy::Stats::SymbolTableImpl symbol_table; + Envoy::Stats::StatNamePool pool(symbol_table); + const std::vector names = prepareK8sClusterStatNames(pool, 200); + + uint32_t index = 0; + for (auto _ : state) { + UNREFERENCED_PARAMETER(_); + benchmark::DoNotOptimize(symbol_table.toString(names[index++ % names.size()])); + } +} +BENCHMARK(bmK8sClusterStatsToString); + +// Encode (string -> StatName) the same mesh-cluster stat names. This exercises +// the well-known encode-map fast path in addTokensToEncoding, where well-known +// tokens are resolved without taking the symbol-table lock. +// NOLINTNEXTLINE(readability-identifier-naming) +static void bmK8sClusterStatsEncode(benchmark::State& state) { + Envoy::Stats::SymbolTableImpl symbol_table; + Envoy::Stats::StatNamePool pool(symbol_table); + // Pre-build the names once to capture their string forms, then re-encode those + // strings on each iteration. + const std::vector names = prepareK8sClusterStatNames(pool, 200); + std::vector strings; + strings.reserve(names.size()); + for (Envoy::Stats::StatName name : names) { + strings.emplace_back(symbol_table.toString(name)); + } + + uint32_t index = 0; + for (auto _ : state) { + UNREFERENCED_PARAMETER(_); + Envoy::Stats::StatNameStorage storage(strings[index++ % strings.size()], symbol_table); + storage.free(symbol_table); + } +} +BENCHMARK(bmK8sClusterStatsEncode); + // --------------------------------------------------------------------------- // decodeNumber micro-benchmarks //