From 3f0b2bd72d2b2e1163c6f2c454c41d2643976269 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Mon, 25 Aug 2025 16:47:22 +0300 Subject: [PATCH 01/48] benchmark update --- pp/wal/benchmarks/scraper_benchmark.cpp | 38 +++++++++++++++++++++++-- pp/wal/hashdex/scraper/scraper.h | 6 ++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/pp/wal/benchmarks/scraper_benchmark.cpp b/pp/wal/benchmarks/scraper_benchmark.cpp index 86f516abcc..ad4925d785 100644 --- a/pp/wal/benchmarks/scraper_benchmark.cpp +++ b/pp/wal/benchmarks/scraper_benchmark.cpp @@ -2,13 +2,40 @@ #include +#include "primitives/timeseries.h" #include "wal/hashdex/scraper/scraper.h" namespace { using PromPP::WAL::hashdex::scraper::PrometheusScraper; -void BenchmarkScraper(benchmark::State& state) { +void BenchmarkScraperParse(benchmark::State& state) { + constexpr auto get_file_name = [] -> std::string { + if (auto& context = benchmark::internal::GetGlobalContext(); context != nullptr) { + return context->operator[]("scraper_file"); + } + + return {}; + }; + + size_t allocated_memory = 0; + + std::ifstream t(get_file_name()); + std::string str((std::istreambuf_iterator(t)), std::istreambuf_iterator()); + + PrometheusScraper scraper; + + for ([[maybe_unused]] auto _ : state) { + auto tmp_str = str; + std::ignore = scraper.parse(tmp_str, 0); + allocated_memory = scraper.allocated_memory(); + } + + state.counters["Alloc"] = benchmark::Counter(allocated_memory, benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024); + ; +} + +void BenchmarkScraperRead(benchmark::State& state) { constexpr auto get_file_name = [] -> std::string { if (auto& context = benchmark::internal::GetGlobalContext(); context != nullptr) { return context->operator[]("scraper_file"); @@ -21,12 +48,17 @@ void BenchmarkScraper(benchmark::State& state) { std::string str((std::istreambuf_iterator(t)), std::istreambuf_iterator()); PrometheusScraper scraper; + std::ignore = scraper.parse(str, 0); for ([[maybe_unused]] auto _ : state) { - std::ignore = scraper.parse(str, 0); + PromPP::Primitives::TimeseriesSemiview ts; + for (auto& metric : scraper.metrics()) { + metric.read(ts); + } } } -BENCHMARK(BenchmarkScraper)->ComputeStatistics("min", [](const std::vector& v) { return *std::ranges::min_element(v); }); +BENCHMARK(BenchmarkScraperParse)->ComputeStatistics("min", [](const std::vector& v) { return *std::ranges::min_element(v); }); +BENCHMARK(BenchmarkScraperRead)->ComputeStatistics("min", [](const std::vector& v) { return *std::ranges::min_element(v); }); } // namespace diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 3f4401d350..4367c212c2 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -100,6 +100,10 @@ class Scraper { [[nodiscard]] PROMPP_ALWAYS_INLINE MetricsWrapper metrics() const noexcept { return MetricsWrapper{*this}; } [[nodiscard]] PROMPP_ALWAYS_INLINE MetadataWrapper metadata() const noexcept { return MetadataWrapper{*this}; } + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { + return metric_buffer_.allocated_memory() + metadata_buffer_.allocated_memory(); + } + private: using Token = Prometheus::textparse::Token; @@ -278,6 +282,8 @@ class Scraper { [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer) const noexcept { return {buffer, this}; } [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return buffer_.allocated_memory(); } + protected: BareBones::Vector buffer_; uint32_t items_count_{}; From 7ae736abb73bcd6fcfe8233b682d89b22d610d91 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Mon, 25 Aug 2025 17:13:16 +0300 Subject: [PATCH 02/48] Added tokenizer traverse benchmark --- pp/wal/benchmarks/scraper_benchmark.cpp | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/pp/wal/benchmarks/scraper_benchmark.cpp b/pp/wal/benchmarks/scraper_benchmark.cpp index ad4925d785..ea6d269f35 100644 --- a/pp/wal/benchmarks/scraper_benchmark.cpp +++ b/pp/wal/benchmarks/scraper_benchmark.cpp @@ -7,8 +7,31 @@ namespace { +using PromPP::WAL::hashdex::scraper::PrometheusParser; using PromPP::WAL::hashdex::scraper::PrometheusScraper; +void BenchmarkParser(benchmark::State& state) { + constexpr auto get_file_name = [] -> std::string { + if (auto& context = benchmark::internal::GetGlobalContext(); context != nullptr) { + return context->operator[]("scraper_file"); + } + + return {}; + }; + + std::ifstream t(get_file_name()); + std::string str((std::istreambuf_iterator(t)), std::istreambuf_iterator()); + + PrometheusParser parser; + + for ([[maybe_unused]] auto _ : state) { + parser.tokenizer().tokenize(str); + while (parser.tokenizer().next() != PromPP::Prometheus::textparse::Token::kEOF) { + benchmark::DoNotOptimize(parser); + } + } +} + void BenchmarkScraperParse(benchmark::State& state) { constexpr auto get_file_name = [] -> std::string { if (auto& context = benchmark::internal::GetGlobalContext(); context != nullptr) { @@ -32,7 +55,6 @@ void BenchmarkScraperParse(benchmark::State& state) { } state.counters["Alloc"] = benchmark::Counter(allocated_memory, benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024); - ; } void BenchmarkScraperRead(benchmark::State& state) { @@ -58,6 +80,7 @@ void BenchmarkScraperRead(benchmark::State& state) { } } +BENCHMARK(BenchmarkParser)->ComputeStatistics("min", [](const std::vector& v) { return *std::ranges::min_element(v); }); BENCHMARK(BenchmarkScraperParse)->ComputeStatistics("min", [](const std::vector& v) { return *std::ranges::min_element(v); }); BENCHMARK(BenchmarkScraperRead)->ComputeStatistics("min", [](const std::vector& v) { return *std::ranges::min_element(v); }); From e49a92f8ff66f4ee30d8e6c8b094d7616e8df75e Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Tue, 26 Aug 2025 10:39:13 +0300 Subject: [PATCH 03/48] small ref --- pp/wal/hashdex/scraper/scraper.h | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 4367c212c2..cd9691b4ed 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -23,10 +23,12 @@ class Scraper { [[nodiscard]] Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { metric_buffer_.initialize(buffer.size() / 2); metadata_buffer_.initialize(buffer.size() / 4); - parser_.tokenizer().tokenize({buffer.data(), buffer.data() + buffer.size()}); + + auto& tokenizer = parser_.tokenizer(); + tokenizer.tokenize({buffer.data(), buffer.data() + buffer.size()}); while (true) { - switch (parser_.tokenizer().next()) { + switch (tokenizer.next()) { case Token::kEOF: case Token::kEOFWord: { return parser_.validate_parse_result(); @@ -55,8 +57,8 @@ class Scraper { return error; } - if (parser_.tokenizer().token() == Token::kExemplar) { - parser_.tokenizer().consume_comment(); + if (tokenizer.token() == Token::kExemplar) { + tokenizer.consume_comment(); } break; @@ -119,14 +121,14 @@ class Scraper { }; } - [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_valid() const noexcept { - return offset != std::numeric_limits::max() && length != std::numeric_limits::max(); + [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_reserved_name() const noexcept { + return offset == std::numeric_limits::max() && length == std::numeric_limits::max(); } [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_empty() const noexcept { return length == 0; } - [[nodiscard]] std::string_view string_view(const std::string_view& buffer) const noexcept { - if (!is_valid()) [[unlikely]] { + [[nodiscard]] std::string_view view(const std::string_view& buffer) const noexcept { + if (is_reserved_name()) [[unlikely]] { return Prometheus::kMetricLabelName; } @@ -145,14 +147,14 @@ class Scraper { PROMPP_ALWAYS_INLINE void sort(const std::string_view& buffer) noexcept { std::sort(labels, labels + count, - [&buffer](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { return a.name.string_view(buffer) < b.name.string_view(buffer); }); + [&buffer](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { return a.name.view(buffer) < b.name.view(buffer); }); } [[nodiscard]] PROMPP_ALWAYS_INLINE uint64_t hash(const std::string_view& buffer) const noexcept { BareBones::XXHash hash; for (uint32_t i = 0; i < count; ++i) { const auto& [name, value] = labels[i]; - hash.extend(name.string_view(buffer), value.string_view(buffer)); + hash.extend(name.view(buffer), value.view(buffer)); } return hash.hash(); } @@ -199,7 +201,7 @@ class Scraper { timeseries.label_set().reserve(item_->label_set.count); for (uint32_t i = 0; i < item_->label_set.count; ++i) { const auto& [name, value] = item_->label_set.labels[i]; - timeseries.label_set().append(name.string_view(buffer_), value.string_view(buffer_)); + timeseries.label_set().append(name.view(buffer_), value.view(buffer_)); } timeseries.samples().emplace_back(item_->sample); @@ -220,8 +222,8 @@ class Scraper { PROMPP_ALWAYS_INLINE void set_item(const MarkedMetadata* item) noexcept { item_ = item; } [[nodiscard]] PROMPP_ALWAYS_INLINE Prometheus::MetadataType type() const noexcept { return item_->type; } - [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view metric_name() const noexcept { return item_->metric_name.string_view(buffer_); } - [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view text() const noexcept { return item_->text.string_view(buffer_); } + [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view metric_name() const noexcept { return item_->metric_name.view(buffer_); } + [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view text() const noexcept { return item_->text.view(buffer_); } private: std::string_view buffer_; From 4656f7137066bf8f873c12459655f1f4a6aa4599 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Fri, 29 Aug 2025 15:19:09 +0300 Subject: [PATCH 04/48] initial encoding --- pp/wal/benchmarks/scraper_benchmark.cpp | 103 ++++----- pp/wal/hashdex/scraper/parser.h | 13 +- pp/wal/hashdex/scraper/scraper.h | 265 ++++++++++++++++++++---- 3 files changed, 282 insertions(+), 99 deletions(-) diff --git a/pp/wal/benchmarks/scraper_benchmark.cpp b/pp/wal/benchmarks/scraper_benchmark.cpp index ea6d269f35..aeaf9fbdd0 100644 --- a/pp/wal/benchmarks/scraper_benchmark.cpp +++ b/pp/wal/benchmarks/scraper_benchmark.cpp @@ -10,27 +10,27 @@ namespace { using PromPP::WAL::hashdex::scraper::PrometheusParser; using PromPP::WAL::hashdex::scraper::PrometheusScraper; -void BenchmarkParser(benchmark::State& state) { - constexpr auto get_file_name = [] -> std::string { - if (auto& context = benchmark::internal::GetGlobalContext(); context != nullptr) { - return context->operator[]("scraper_file"); - } - - return {}; - }; - - std::ifstream t(get_file_name()); - std::string str((std::istreambuf_iterator(t)), std::istreambuf_iterator()); - - PrometheusParser parser; - - for ([[maybe_unused]] auto _ : state) { - parser.tokenizer().tokenize(str); - while (parser.tokenizer().next() != PromPP::Prometheus::textparse::Token::kEOF) { - benchmark::DoNotOptimize(parser); - } - } -} +// void BenchmarkParser(benchmark::State& state) { +// constexpr auto get_file_name = [] -> std::string { +// if (auto& context = benchmark::internal::GetGlobalContext(); context != nullptr) { +// return context->operator[]("scraper_file"); +// } +// +// return {}; +// }; +// +// std::ifstream t(get_file_name()); +// std::string str((std::istreambuf_iterator(t)), std::istreambuf_iterator()); +// +// PrometheusParser parser; +// +// for ([[maybe_unused]] auto _ : state) { +// parser.tokenizer().tokenize(str); +// while (parser.tokenizer().next() != PromPP::Prometheus::textparse::Token::kEOF) { +// benchmark::DoNotOptimize(parser); +// } +// } +// } void BenchmarkScraperParse(benchmark::State& state) { constexpr auto get_file_name = [] -> std::string { @@ -41,47 +41,48 @@ void BenchmarkScraperParse(benchmark::State& state) { return {}; }; - size_t allocated_memory = 0; - std::ifstream t(get_file_name()); std::string str((std::istreambuf_iterator(t)), std::istreambuf_iterator()); - PrometheusScraper scraper; - for ([[maybe_unused]] auto _ : state) { + PrometheusScraper scraper; auto tmp_str = str; std::ignore = scraper.parse(tmp_str, 0); - allocated_memory = scraper.allocated_memory(); } - state.counters["Alloc"] = benchmark::Counter(allocated_memory, benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024); -} - -void BenchmarkScraperRead(benchmark::State& state) { - constexpr auto get_file_name = [] -> std::string { - if (auto& context = benchmark::internal::GetGlobalContext(); context != nullptr) { - return context->operator[]("scraper_file"); - } - - return {}; - }; - - std::ifstream t(get_file_name()); - std::string str((std::istreambuf_iterator(t)), std::istreambuf_iterator()); - - PrometheusScraper scraper; - std::ignore = scraper.parse(str, 0); - - for ([[maybe_unused]] auto _ : state) { - PromPP::Primitives::TimeseriesSemiview ts; - for (auto& metric : scraper.metrics()) { - metric.read(ts); - } + { + PrometheusScraper scraper; + auto tmp_str = str; + std::ignore = scraper.parse(tmp_str, 0); + state.counters["Alloc"] = benchmark::Counter(scraper.allocated_memory(), benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024); } } -BENCHMARK(BenchmarkParser)->ComputeStatistics("min", [](const std::vector& v) { return *std::ranges::min_element(v); }); +// void BenchmarkScraperRead(benchmark::State& state) { +// constexpr auto get_file_name = [] -> std::string { +// if (auto& context = benchmark::internal::GetGlobalContext(); context != nullptr) { +// return context->operator[]("scraper_file"); +// } +// +// return {}; +// }; +// +// std::ifstream t(get_file_name()); +// std::string str((std::istreambuf_iterator(t)), std::istreambuf_iterator()); +// +// PrometheusScraper scraper; +// std::ignore = scraper.parse(str, 0); +// +// for ([[maybe_unused]] auto _ : state) { +// PromPP::Primitives::TimeseriesSemiview ts; +// for (auto& metric : scraper.metrics()) { +// metric.read(ts); +// } +// } +// } + +// BENCHMARK(BenchmarkParser)->ComputeStatistics("min", [](const std::vector& v) { return *std::ranges::min_element(v); }); BENCHMARK(BenchmarkScraperParse)->ComputeStatistics("min", [](const std::vector& v) { return *std::ranges::min_element(v); }); -BENCHMARK(BenchmarkScraperRead)->ComputeStatistics("min", [](const std::vector& v) { return *std::ranges::min_element(v); }); +// BENCHMARK(BenchmarkScraperRead)->ComputeStatistics("min", [](const std::vector& v) { return *std::ranges::min_element(v); }); } // namespace diff --git a/pp/wal/hashdex/scraper/parser.h b/pp/wal/hashdex/scraper/parser.h index c539cb5941..76e03bdde6 100644 --- a/pp/wal/hashdex/scraper/parser.h +++ b/pp/wal/hashdex/scraper/parser.h @@ -31,13 +31,13 @@ template } template -concept ParserInterface = requires(Parser& parser, const Parser& const_parser, Primitives::Timestamp& timestamp) { +concept ParserInterface = requires(Parser& parser, const Parser& const_parser, Primitives::Timestamp& timestamp, bool& has_timestamp) { { parser.tokenizer() }; { Prometheus::textparse::TokenizerInterface }; { Prometheus::textparse::TokenizerInterface }; { const_parser.is_value_token() } -> std::same_as; - { parser.parse_timestamp(timestamp) } -> std::same_as; + { parser.parse_timestamp(timestamp, has_timestamp) } -> std::same_as; { const_parser.validate_parse_result() } -> std::same_as; { const_parser.validate_parse_sample_result() } -> std::same_as; }; @@ -59,13 +59,15 @@ class PrometheusParser { return (tokenizer_.token() == Token::kLinebreak || tokenizer_.token() == Token::kEOF) ? Error::kNoError : Error::kUnexpectedToken; } - [[nodiscard]] Error parse_timestamp(Primitives::Timestamp& timestamp) noexcept { + [[nodiscard]] Error parse_timestamp(Primitives::Timestamp& timestamp, bool& has_timestamp) noexcept { + has_timestamp = false; if (is_timestamp_token()) { if (!parse_numeric_value(tokenizer_.token_str(), timestamp)) [[unlikely]] { return Error::kInvalidTimestamp; } tokenizer_.next_non_whitespace(); + has_timestamp = true; } return Error::kNoError; @@ -96,7 +98,8 @@ class OpenMetricsParser { return (tokenizer_.token() == Token::kLinebreak || tokenizer_.token() == Token::kExemplar) ? Error::kNoError : Error::kUnexpectedToken; } - [[nodiscard]] Error parse_timestamp(Primitives::Timestamp& timestamp) noexcept { + [[nodiscard]] Error parse_timestamp(Primitives::Timestamp& timestamp, bool& has_timestamp) noexcept { + has_timestamp = false; if (tokenizer_.token() == Token::kTimestamp) { if (double float_timestamp; parse_timestamp_as_float(float_timestamp)) [[likely]] { timestamp = static_cast(float_timestamp * 1000.0); @@ -105,8 +108,8 @@ class OpenMetricsParser { } tokenizer_.next_non_whitespace(); + has_timestamp = true; } - return Error::kNoError; } diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index cd9691b4ed..b0486f8cbd 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -2,6 +2,7 @@ #include +#include #include #include "bare_bones/algorithm.h" @@ -21,8 +22,7 @@ template class Scraper { public: [[nodiscard]] Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { - metric_buffer_.initialize(buffer.size() / 2); - metadata_buffer_.initialize(buffer.size() / 4); + labels_.reserve(255); auto& tokenizer = parser_.tokenizer(); tokenizer.tokenize({buffer.data(), buffer.data() + buffer.size()}); @@ -51,8 +51,8 @@ class Scraper { case Token::kMetricName: case Token::kBraceOpen: { - if (const auto error = MetricParser{parser_, metric_buffer_, metric_buffer_.add_metric(default_timestamp)}.parse(); error != Error::kNoError) - [[unlikely]] { + if (const auto error = MetricParser{parser_, metric_buffer_, metric_buffer_.add_metric(), labels_, default_timestamp}.parse(); + error != Error::kNoError) [[unlikely]] { metric_buffer_.remove_item(); return error; } @@ -141,38 +141,20 @@ class Scraper { MarkedString value; }; - struct MarkedLabelSet { - uint32_t count{}; - MarkedLabel labels[]; - - PROMPP_ALWAYS_INLINE void sort(const std::string_view& buffer) noexcept { - std::sort(labels, labels + count, - [&buffer](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { return a.name.view(buffer) < b.name.view(buffer); }); - } - - [[nodiscard]] PROMPP_ALWAYS_INLINE uint64_t hash(const std::string_view& buffer) const noexcept { - BareBones::XXHash hash; - for (uint32_t i = 0; i < count; ++i) { - const auto& [name, value] = labels[i]; - hash.extend(name.view(buffer), value.view(buffer)); - } - return hash.hash(); - } + struct MarkedSample { + Primitives::Sample sample{}; + bool has_ts{}; }; struct MarkedMetric { uint64_t hash{}; - Primitives::Sample sample{}; - MarkedLabelSet label_set; + uint32_t offset{}; + uint32_t offset_extra{}; + uint8_t bytes[]; - explicit MarkedMetric(Primitives::Timestamp timestamp) : sample(timestamp, 0.0) {} + // explicit MarkedMetric(Primitives::Timestamp timestamp) : sample(timestamp, 0.0) {} - PROMPP_ALWAYS_INLINE void calculate_hash(const std::string_view& buffer) noexcept { - label_set.sort(buffer); - hash = label_set.hash(buffer); - } - - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t occupied_size() const noexcept { return sizeof(*this) + sizeof(MarkedLabel) * label_set.count; } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t occupied_size() const noexcept { return sizeof(*this) + sizeof(MarkedLabel) /*add bytes skip!!!*/; } }; struct MarkedMetadata { @@ -293,12 +275,52 @@ class Scraper { class MetricMarkupBuffer : public MarkupBuffer { public: - PROMPP_ALWAYS_INLINE MarkedMetric* add_metric(Primitives::Timestamp default_timestamp) noexcept { + PROMPP_ALWAYS_INLINE MarkedMetric* add_metric() noexcept { ++this->items_count_; const auto offset = this->buffer_.size(); this->buffer_.resize(offset + sizeof(MarkedMetric)); - return new (reinterpret_cast(this->buffer_.data() + offset)) MarkedMetric(default_timestamp); + return std::construct_at(reinterpret_cast(this->buffer_.data() + offset)); + } + + PROMPP_ALWAYS_INLINE void add_count(uint32_t count, MarkedMetric*& metric) noexcept { + constexpr uint32_t kVarint1Byte = 0x80; // 2^7 + constexpr uint32_t kVarint2Byte = 0x4000; // 2^14 + constexpr uint32_t kVarint3Byte = 0x200000; // 2^21 + constexpr uint32_t kVarint4Byte = 0x10000000; // 2^28 + constexpr uint8_t kContinueBit = 0x80; + constexpr uint8_t kValueMask = 0x7F; + + const auto offset = reinterpret_cast(metric) - this->buffer_.data(); + + std::array tmp{}; + char* out = tmp.data(); + + if (count < kVarint1Byte) { + *out++ = static_cast(count); + } else if (count < kVarint2Byte) { + *out++ = static_cast((count & kValueMask) | kContinueBit); + *out++ = static_cast(count >> 7); + } else if (count < kVarint3Byte) { + *out++ = static_cast((count & kValueMask) | kContinueBit); + *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); + *out++ = static_cast(count >> 14); + } else if (count < kVarint4Byte) { + *out++ = static_cast((count & kValueMask) | kContinueBit); + *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); + *out++ = static_cast(((count >> 14) & kValueMask) | kContinueBit); + *out++ = static_cast(count >> 21); + } else { + *out++ = static_cast((count & kValueMask) | kContinueBit); + *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); + *out++ = static_cast(((count >> 14) & kValueMask) | kContinueBit); + *out++ = static_cast(((count >> 21) & kValueMask) | kContinueBit); + *out++ = static_cast(count >> 28); + } + + this->buffer_.push_back(tmp.data(), out); + + metric = reinterpret_cast(this->buffer_.data() + offset); } PROMPP_ALWAYS_INLINE void add_label(const MarkedLabel& label, MarkedMetric*& metric) noexcept { @@ -306,10 +328,116 @@ class Scraper { return; } + auto name = label.name; + if (name.is_reserved_name()) [[unlikely]] { + name.offset = 0; + name.length = 0; + } + + const uint8_t sz0 = encode_size(name.offset); + const uint8_t sz1 = encode_size(name.length); + const uint8_t sz2 = encode_size(label.value.offset); + const uint8_t sz3 = encode_size(label.value.length); + + const uint8_t layout = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); + + std::array tmp{}; + char* out = tmp.data(); + + *out++ = static_cast(layout); + + *reinterpret_cast(out) = name.offset; + out += sz0 + 1; + + *reinterpret_cast(out) = name.length; + out += sz1 + 1; + + *reinterpret_cast(out) = label.value.offset; + out += sz2 + 1; + + *reinterpret_cast(out) = label.value.length; + out += sz3 + 1; + const auto offset = reinterpret_cast(metric) - this->buffer_.data(); - this->buffer_.push_back(reinterpret_cast(&label), reinterpret_cast(&label) + sizeof(label)); + + this->buffer_.push_back(tmp.data(), out); + metric = reinterpret_cast(this->buffer_.data() + offset); - ++metric->label_set.count; + } + + PROMPP_ALWAYS_INLINE void add_sample(const MarkedSample& sample, MarkedMetric*& metric) noexcept { + const auto offset = reinterpret_cast(metric) - this->buffer_.data(); + + add_sample_internal(sample); + + metric = reinterpret_cast(this->buffer_.data() + offset); + } + + PROMPP_ALWAYS_INLINE void add_sample_internal(const MarkedSample& sample) noexcept { + uint8_t marker = sample.has_ts ? 0b10000000 : 0; + const double val = sample.sample.value(); + + auto flush = [&](uint8_t m) PROMPP_LAMBDA_INLINE { + this->buffer_.push_back(marker | m); + if (sample.has_ts) { + append(sample.sample.timestamp()); + } + }; + + if (std::isnan(val)) [[unlikely]] { + flush(0b00000100); // staleNaN + return; + } + + if (val == 0.0) [[unlikely]] { + flush(0b00000000); // zero + return; + } + + if (std::trunc(val) == val && val > 0.0) [[likely]] { // u integer + auto ival = static_cast(val); + if (ival <= std::numeric_limits::max()) [[unlikely]] { + flush(0b00000001); + append(static_cast(ival)); + return; + } + if (ival <= std::numeric_limits::max()) [[unlikely]] { + flush(0b00000010); + append(static_cast(ival)); + return; + } + if (ival <= std::numeric_limits::max()) [[likely]] { + flush(0b00000011); + append(static_cast(ival)); + return; + } + } + + float f = static_cast(val); + if (static_cast(f) == val) [[unlikely]] { + flush(0b00001000); // float32 + append(f); + } else { + flush(0b00001001); // double + append(val); + } + } + + private: + template + PROMPP_ALWAYS_INLINE void append(const T val) noexcept { + auto p = reinterpret_cast(&val); + this->buffer_.push_back(p, p + sizeof(T)); + } + + static uint8_t encode_size(uint32_t v) noexcept { + if (v <= 0xFF) + return 0; + if (v <= 0xFFFF) + return 1; + if (v <= 0xFFFFFF) + return 2; + return 3; } }; @@ -330,13 +458,29 @@ class Scraper { class MetricParser { public: - MetricParser(Parser& parser, MetricMarkupBuffer& markup_buffer, MarkedMetric* metric) : parser_(parser), markup_buffer_(markup_buffer), metric_(metric) {} + MetricParser(Parser& parser, + MetricMarkupBuffer& markup_buffer, + MarkedMetric* metric, + BareBones::Vector& labels, + Primitives::Timestamp timestamp) + : parser_(parser), markup_buffer_(markup_buffer), metric_(metric), labels_(labels), sample_{.sample = {timestamp, 0.0}} {} [[nodiscard]] Error parse() noexcept { + labels_.clear(); + bool have_metric_name = false; auto& tokenizer = parser_.tokenizer(); + + metric_->offset = tokenizer.token_str().data() - tokenizer.buffer().data(); + if (tokenizer.token() == Token::kMetricName) [[likely]] { - markup_buffer_.add_label(MarkedLabel{.value = MarkedString::create(tokenizer.token_str(), tokenizer.buffer())}, metric_); + // markup_buffer_.add_label(MarkedLabel{.value = MarkedString::create(tokenizer.token_str(), tokenizer.buffer())}, metric_); + + auto string = MarkedString::create(tokenizer.token_str(), tokenizer.buffer()); + string.offset -= metric_->offset; + auto label = MarkedLabel{.value = string}; + labels_.push_back(label); + have_metric_name = true; tokenizer.next_non_whitespace(); } else if (tokenizer.token() == Token::kWhitespace) [[likely]] { @@ -357,7 +501,31 @@ class Scraper { return Error::kNoMetricName; } - metric_->calculate_hash(tokenizer.buffer()); + // metric_->calculate_hash(tokenizer.buffer()); + // sort + std::sort(labels_.begin(), labels_.end(), [buffer = tokenizer.buffer()](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { + return a.name.view(buffer) < b.name.view(buffer); + }); + + // hash + { + BareBones::XXHash hash; + for (const auto& label : labels_) { + hash.extend(label.name.view(tokenizer.buffer()), label.value.view(tokenizer.buffer())); + } + metric_->hash = hash.hash(); + } + + // encode count + { + markup_buffer_.add_count(labels_.size(), metric_); + } + + // encode labels + for (const auto& label : labels_) { + markup_buffer_.add_label(label, metric_); + } + return parse_metric_suffix(); } @@ -365,6 +533,8 @@ class Scraper { Parser& parser_; MetricMarkupBuffer& markup_buffer_; MarkedMetric* metric_; + BareBones::Vector& labels_; + MarkedSample sample_; [[nodiscard]] Error tokenize_label_set(bool& have_metric_name) noexcept { auto& tokenizer = parser_.tokenizer(); @@ -385,11 +555,15 @@ class Scraper { return error; } - markup_buffer_.add_label(label, metric_); + // markup_buffer_.add_label(label, metric_); + labels_.push_back(label); + tokenizer.next(); } else { if (!have_metric_name) [[unlikely]] { - markup_buffer_.add_label(MarkedLabel{.value = label.name}, metric_); + // markup_buffer_.add_label(MarkedLabel{.value = label.name}, metric_); + labels_.push_back(label); + have_metric_name = true; } else { return Error::kUnexpectedToken; @@ -411,6 +585,7 @@ class Scraper { if (tokenizer.token() == Token::kLabelName) [[likely]] { label_name = MarkedString::create(tokenizer.token_str(), tokenizer.buffer()); + label_name.offset -= metric_->offset; return Error::kNoError; } if (tokenizer.token() == Token::kQuotedString) { @@ -441,6 +616,7 @@ class Scraper { } string = MarkedString::create(value, tokenizer.buffer()); + string.offset -= metric_->offset; return Error::kNoError; } @@ -453,22 +629,24 @@ class Scraper { return error; } + markup_buffer_.add_sample(sample_, metric_); + return parser_.validate_parse_sample_result(); } [[nodiscard]] Error parse_sample() noexcept { auto& tokenizer = parser_.tokenizer(); - if (!parse_numeric_value(tokenizer.token_str(), metric_->sample.value())) [[unlikely]] { + if (!parse_numeric_value(tokenizer.token_str(), sample_.sample.value())) [[unlikely]] { return Error::kInvalidValue; } - if (std::isnan(metric_->sample.value())) [[unlikely]] { - metric_->sample.value() = Prometheus::kNormalNan; + if (std::isnan(sample_.sample.value())) [[unlikely]] { + sample_.sample.value() = Prometheus::kNormalNan; } tokenizer.next_non_whitespace(); - return parser_.parse_timestamp(metric_->sample.timestamp()); + return parser_.parse_timestamp(sample_.sample.timestamp(), sample_.has_ts); } }; @@ -515,6 +693,7 @@ class Scraper { Parser parser_; MetricMarkupBuffer metric_buffer_; MetadataMarkupBuffer metadata_buffer_; + BareBones::Vector labels_; }; using PrometheusScraper = Scraper; From 0027219f0e151a927c1071add24f4b2b136ec280 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Sat, 30 Aug 2025 22:36:08 +0300 Subject: [PATCH 05/48] read + passing all tests --- pp/bare_bones/vector.h | 6 + pp/wal/benchmarks/scraper_benchmark.cpp | 92 +++---- pp/wal/hashdex/scraper/scraper.h | 327 ++++++++++++++++++------ 3 files changed, 297 insertions(+), 128 deletions(-) diff --git a/pp/bare_bones/vector.h b/pp/bare_bones/vector.h index cd58598b96..221926f69a 100644 --- a/pp/bare_bones/vector.h +++ b/pp/bare_bones/vector.h @@ -108,6 +108,12 @@ class GenericVector { derived()->set_size(0); } + PROMPP_ALWAYS_INLINE void pop_back() noexcept { + assert(!empty()); + + erase(end() - 1, end()); + } + PROMPP_ALWAYS_INLINE iterator erase(iterator first, iterator last) noexcept { assert(first >= begin()); assert(last <= end()); diff --git a/pp/wal/benchmarks/scraper_benchmark.cpp b/pp/wal/benchmarks/scraper_benchmark.cpp index aeaf9fbdd0..152079318b 100644 --- a/pp/wal/benchmarks/scraper_benchmark.cpp +++ b/pp/wal/benchmarks/scraper_benchmark.cpp @@ -10,27 +10,27 @@ namespace { using PromPP::WAL::hashdex::scraper::PrometheusParser; using PromPP::WAL::hashdex::scraper::PrometheusScraper; -// void BenchmarkParser(benchmark::State& state) { -// constexpr auto get_file_name = [] -> std::string { -// if (auto& context = benchmark::internal::GetGlobalContext(); context != nullptr) { -// return context->operator[]("scraper_file"); -// } -// -// return {}; -// }; -// -// std::ifstream t(get_file_name()); -// std::string str((std::istreambuf_iterator(t)), std::istreambuf_iterator()); -// -// PrometheusParser parser; -// -// for ([[maybe_unused]] auto _ : state) { -// parser.tokenizer().tokenize(str); -// while (parser.tokenizer().next() != PromPP::Prometheus::textparse::Token::kEOF) { -// benchmark::DoNotOptimize(parser); -// } -// } -// } +void BenchmarkParser(benchmark::State& state) { + constexpr auto get_file_name = [] -> std::string { + if (auto& context = benchmark::internal::GetGlobalContext(); context != nullptr) { + return context->operator[]("scraper_file"); + } + + return {}; + }; + + std::ifstream t(get_file_name()); + std::string str((std::istreambuf_iterator(t)), std::istreambuf_iterator()); + + PrometheusParser parser; + + for ([[maybe_unused]] auto _ : state) { + parser.tokenizer().tokenize(str); + while (parser.tokenizer().next() != PromPP::Prometheus::textparse::Token::kEOF) { + benchmark::DoNotOptimize(parser); + } + } +} void BenchmarkScraperParse(benchmark::State& state) { constexpr auto get_file_name = [] -> std::string { @@ -58,31 +58,31 @@ void BenchmarkScraperParse(benchmark::State& state) { } } -// void BenchmarkScraperRead(benchmark::State& state) { -// constexpr auto get_file_name = [] -> std::string { -// if (auto& context = benchmark::internal::GetGlobalContext(); context != nullptr) { -// return context->operator[]("scraper_file"); -// } -// -// return {}; -// }; -// -// std::ifstream t(get_file_name()); -// std::string str((std::istreambuf_iterator(t)), std::istreambuf_iterator()); -// -// PrometheusScraper scraper; -// std::ignore = scraper.parse(str, 0); -// -// for ([[maybe_unused]] auto _ : state) { -// PromPP::Primitives::TimeseriesSemiview ts; -// for (auto& metric : scraper.metrics()) { -// metric.read(ts); -// } -// } -// } - -// BENCHMARK(BenchmarkParser)->ComputeStatistics("min", [](const std::vector& v) { return *std::ranges::min_element(v); }); +void BenchmarkScraperRead(benchmark::State& state) { + constexpr auto get_file_name = [] -> std::string { + if (auto& context = benchmark::internal::GetGlobalContext(); context != nullptr) { + return context->operator[]("scraper_file"); + } + + return {}; + }; + + std::ifstream t(get_file_name()); + std::string str((std::istreambuf_iterator(t)), std::istreambuf_iterator()); + + PrometheusScraper scraper; + std::ignore = scraper.parse(str, 0); + + for ([[maybe_unused]] auto _ : state) { + PromPP::Primitives::TimeseriesSemiview ts; + for (auto& metric : scraper.metrics()) { + metric.read(ts); + } + } +} + +BENCHMARK(BenchmarkParser)->ComputeStatistics("min", [](const std::vector& v) { return *std::ranges::min_element(v); }); BENCHMARK(BenchmarkScraperParse)->ComputeStatistics("min", [](const std::vector& v) { return *std::ranges::min_element(v); }); -// BENCHMARK(BenchmarkScraperRead)->ComputeStatistics("min", [](const std::vector& v) { return *std::ranges::min_element(v); }); +BENCHMARK(BenchmarkScraperRead)->ComputeStatistics("min", [](const std::vector& v) { return *std::ranges::min_element(v); }); } // namespace diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index b0486f8cbd..660f13e14e 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -23,6 +23,7 @@ class Scraper { public: [[nodiscard]] Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { labels_.reserve(255); + default_timestamp_ = default_timestamp; auto& tokenizer = parser_.tokenizer(); tokenizer.tokenize({buffer.data(), buffer.data() + buffer.size()}); @@ -51,7 +52,9 @@ class Scraper { case Token::kMetricName: case Token::kBraceOpen: { - if (const auto error = MetricParser{parser_, metric_buffer_, metric_buffer_.add_metric(), labels_, default_timestamp}.parse(); + if (const auto error = MetricParser{parser_, metric_buffer_, labels_, static_cast(tokenizer.token_str().data() - tokenizer.buffer().data()), + default_timestamp} + .parse(); error != Error::kNoError) [[unlikely]] { metric_buffer_.remove_item(); return error; @@ -76,7 +79,9 @@ class Scraper { explicit MetricsWrapper(const Scraper& scraper) : scraper_(scraper) {} [[nodiscard]] PROMPP_LAMBDA_INLINE uint32_t size() const noexcept { return scraper_.metric_buffer_.items_count(); } - [[nodiscard]] PROMPP_LAMBDA_INLINE auto begin() const noexcept { return scraper_.metric_buffer_.begin(scraper_.parser_.tokenizer().buffer()); } + [[nodiscard]] PROMPP_LAMBDA_INLINE auto begin() const noexcept { + return scraper_.metric_buffer_.begin(scraper_.parser_.tokenizer().buffer(), scraper_.default_timestamp()); + } [[nodiscard]] PROMPP_LAMBDA_INLINE static auto end() noexcept { return MetricMarkupBuffer::end(); } private: @@ -96,12 +101,14 @@ class Scraper { }; [[nodiscard]] PROMPP_LAMBDA_INLINE uint32_t size() const noexcept { return metric_buffer_.items_count(); } - [[nodiscard]] PROMPP_LAMBDA_INLINE auto begin() const noexcept { return metric_buffer_.begin(parser_.tokenizer().buffer()); } + [[nodiscard]] PROMPP_LAMBDA_INLINE auto begin() const noexcept { return metric_buffer_.begin(parser_.tokenizer().buffer(), default_timestamp_); } [[nodiscard]] PROMPP_LAMBDA_INLINE static auto end() noexcept { return MetricMarkupBuffer::end(); } [[nodiscard]] PROMPP_ALWAYS_INLINE MetricsWrapper metrics() const noexcept { return MetricsWrapper{*this}; } [[nodiscard]] PROMPP_ALWAYS_INLINE MetadataWrapper metadata() const noexcept { return MetadataWrapper{*this}; } + [[nodiscard]] PROMPP_ALWAYS_INLINE Primitives::Timestamp default_timestamp() const noexcept { return default_timestamp_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return metric_buffer_.allocated_memory() + metadata_buffer_.allocated_memory(); } @@ -148,21 +155,14 @@ class Scraper { struct MarkedMetric { uint64_t hash{}; - uint32_t offset{}; - uint32_t offset_extra{}; - uint8_t bytes[]; - - // explicit MarkedMetric(Primitives::Timestamp timestamp) : sample(timestamp, 0.0) {} - - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t occupied_size() const noexcept { return sizeof(*this) + sizeof(MarkedLabel) /*add bytes skip!!!*/; } + uint32_t base_offset{}; + uint32_t data_offset{}; }; struct MarkedMetadata { MarkedString metric_name{}; MarkedString text{}; Prometheus::MetadataType type{}; - - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t occupied_size() const noexcept { return sizeof(*this); } }; #pragma pack(pop) @@ -171,7 +171,8 @@ class Scraper { public: using MarkedItem = MarkedMetric; - explicit Metric(std::string_view buffer, const MarkedMetric* item) : buffer_(buffer), item_(item) {} + Metric(std::string_view buffer, const BareBones::Vector& bytes_buffer, const MarkedMetric* item, Primitives::Timestamp default_timestamp) + : buffer_(buffer), bytes_buffer_(bytes_buffer), item_(item), default_timestamp_(default_timestamp) {} [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetric* item() const noexcept { return item_; } PROMPP_ALWAYS_INLINE void set_item(const MarkedMetric* item) noexcept { item_ = item; } @@ -179,19 +180,141 @@ class Scraper { [[nodiscard]] PROMPP_ALWAYS_INLINE uint64_t hash() const noexcept { return item_->hash; } template - void read(Timeseries& timeseries) const { - timeseries.label_set().reserve(item_->label_set.count); - for (uint32_t i = 0; i < item_->label_set.count; ++i) { - const auto& [name, value] = item_->label_set.labels[i]; - timeseries.label_set().append(name.view(buffer_), value.view(buffer_)); + void read(Timeseries& ts) const { + const char* ptr = bytes_buffer_.data() + item_->data_offset; + const char* base = buffer_.data() + item_->base_offset; + + // decode label count + uint32_t labels_count = decode_varint(ptr); + ts.label_set().reserve(labels_count); + + // decode label + for (uint32_t i = 0; i < labels_count; ++i) { + const uint8_t layout = static_cast(*ptr++); + const uint8_t sz0 = (layout >> 0) & 0x3; + const uint8_t sz1 = (layout >> 2) & 0x3; + const uint8_t sz2 = (layout >> 4) & 0x3; + const uint8_t sz3 = (layout >> 6) & 0x3; + + auto read_val = [&](uint8_t sz) PROMPP_LAMBDA_INLINE -> uint32_t { + uint32_t v = 0; + memcpy(&v, ptr, sz + 1); + ptr += sz + 1; + return v; + }; + + uint32_t name_off = read_val(sz0); + uint32_t name_len = read_val(sz1); + uint32_t value_off = read_val(sz2); + uint32_t value_len = read_val(sz3); + + if (name_off == 0 && name_len == 0) [[unlikely]] { + ts.label_set().append(Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); + } else { + ts.label_set().append(std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); + } + } + + // decode sample + uint8_t marker = static_cast(*ptr++); + bool has_ts = (marker & 0b10000000) != 0; + uint8_t type = marker & 0b01111111; + + Primitives::Sample sample{}; + sample.timestamp() = default_timestamp_; + + if (has_ts) [[unlikely]] { + int64_t ts_val; + memcpy(&ts_val, ptr, sizeof(ts_val)); + ptr += sizeof(ts_val); + sample.timestamp() = ts_val; } - timeseries.samples().emplace_back(item_->sample); + double val; + + switch (type) { + case 0b00000011: { // uint32 + uint32_t tmp; + memcpy(&tmp, ptr, sizeof(tmp)); + ptr += sizeof(tmp); + val = static_cast(tmp); + break; + } + case 0b00000000: // zero + val = 0.0; + break; + case 0b00000001: { // uint8 + uint8_t tmp = static_cast(*ptr++); + val = static_cast(tmp); + break; + } + case 0b00000010: { // uint16 + uint16_t tmp; + memcpy(&tmp, ptr, sizeof(tmp)); + ptr += sizeof(tmp); + val = static_cast(tmp); + break; + } + case 0b00000100: // NaN + val = Prometheus::kNormalNan; + break; + case 0b00001000: { // float32 + float tmp; + memcpy(&tmp, ptr, sizeof(tmp)); + ptr += sizeof(tmp); + val = static_cast(tmp); + break; + } + case 0b00001001: { // double + double tmp; + memcpy(&tmp, ptr, sizeof(tmp)); + ptr += sizeof(tmp); + val = tmp; + break; + } + default: + val = Prometheus::kStaleNan; + break; + } + + sample.value() = val; + ts.samples().emplace_back(sample); } private: std::string_view buffer_; + const BareBones::Vector& bytes_buffer_; const MarkedMetric* item_{}; + Primitives::Timestamp default_timestamp_; + + static uint32_t decode_varint(const char*& ptr) noexcept { + uint32_t b0 = static_cast(*ptr++); + if ((b0 & 0x80) == 0) [[likely]] { + return b0; + } + + uint32_t b1 = static_cast(*ptr++); + uint32_t v = (b0 & 0x7F) | ((b1 & 0x7F) << 7); + if ((b1 & 0x80) == 0) { + return v; + } + + uint32_t b2 = static_cast(*ptr++); + v |= (b2 & 0x7F) << 14; + if ((b2 & 0x80) == 0) { + return v; + } + + uint32_t b3 = static_cast(*ptr++); + v |= (b3 & 0x7F) << 21; + if ((b3 & 0x80) == 0) { + return v; + } + + uint32_t b4 = static_cast(*ptr++); + v |= (b4 & 0x0F) << 28; + return v; + } }; class Metadata { @@ -234,7 +357,7 @@ class Scraper { [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type* operator->() const noexcept { return &item_; } PROMPP_ALWAYS_INLINE Iterator& operator++() noexcept { - item_.set_item(reinterpret_cast(reinterpret_cast(item_.item()) + item_.item()->occupied_size())); + item_.set_item(reinterpret_cast(reinterpret_cast(item_.item()) + sizeof(*item_.item()))); --items_count_; return *this; } @@ -273,17 +396,75 @@ class Scraper { uint32_t items_count_{}; }; - class MetricMarkupBuffer : public MarkupBuffer { + class MetricMarkupBuffer { public: - PROMPP_ALWAYS_INLINE MarkedMetric* add_metric() noexcept { - ++this->items_count_; + class IteratorSentinel {}; - const auto offset = this->buffer_.size(); - this->buffer_.resize(offset + sizeof(MarkedMetric)); - return std::construct_at(reinterpret_cast(this->buffer_.data() + offset)); + class Iterator { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = Metric; + using difference_type = ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + Iterator(std::string_view buffer, + const BareBones::Vector& bytes_buffer, + const MarkedMetric* ptr, + uint32_t items_count, + Primitives::Timestamp default_timestamp) + : item_(buffer, bytes_buffer, ptr, default_timestamp), ptr_(ptr), buffer_(buffer), bytes_buffer_(bytes_buffer), items_count_(items_count) {} + + [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type& operator*() const noexcept { return item_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type* operator->() const noexcept { return &item_; } + + PROMPP_ALWAYS_INLINE Iterator& operator++() noexcept { + item_.set_item(++ptr_); + --items_count_; + return *this; + } + + PROMPP_ALWAYS_INLINE Iterator operator++(int) noexcept { + auto tmp = *this; + ++(*this); + return tmp; + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE bool operator==(const IteratorSentinel&) const noexcept { return items_count_ == 0; } + + private: + Metric item_; + const MarkedMetric* ptr_{}; + std::string_view buffer_; + const BareBones::Vector& bytes_buffer_; + uint32_t items_count_{}; + }; + + [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer, Primitives::Timestamp default_timestamp) const noexcept { + return {buffer, bytes_buffer_, metric_buffer_.data(), metric_buffer_.size(), default_timestamp}; } - PROMPP_ALWAYS_INLINE void add_count(uint32_t count, MarkedMetric*& metric) noexcept { + [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } + + [[nodiscard]] PROMPP_ALWAYS_INLINE const BareBones::Vector& metric_buffer() const noexcept { return metric_buffer_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE const BareBones::Vector& bytes_buffer() const noexcept { return bytes_buffer_; } + + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t items_count() const noexcept { return metric_buffer_.size(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t bytes_count() const noexcept { return bytes_buffer_.size(); } + + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return metric_buffer_.allocated_memory() + bytes_buffer_.allocated_memory(); } + + PROMPP_ALWAYS_INLINE void remove_item() noexcept { + bytes_buffer_.resize(metric_buffer_.back().data_offset); + metric_buffer_.pop_back(); + } + + PROMPP_ALWAYS_INLINE void add_hash(uint64_t hash) noexcept { metric_buffer_.back().hash = hash; } + PROMPP_ALWAYS_INLINE void add_metric(uint32_t global_offset) noexcept { + metric_buffer_.push_back(MarkedMetric{.base_offset = global_offset, .data_offset = bytes_count()}); + } + + PROMPP_ALWAYS_INLINE void add_count(uint32_t count) noexcept { constexpr uint32_t kVarint1Byte = 0x80; // 2^7 constexpr uint32_t kVarint2Byte = 0x4000; // 2^14 constexpr uint32_t kVarint3Byte = 0x200000; // 2^21 @@ -291,8 +472,6 @@ class Scraper { constexpr uint8_t kContinueBit = 0x80; constexpr uint8_t kValueMask = 0x7F; - const auto offset = reinterpret_cast(metric) - this->buffer_.data(); - std::array tmp{}; char* out = tmp.data(); @@ -318,24 +497,22 @@ class Scraper { *out++ = static_cast(count >> 28); } - this->buffer_.push_back(tmp.data(), out); - - metric = reinterpret_cast(this->buffer_.data() + offset); + bytes_buffer_.push_back(tmp.data(), out); } - PROMPP_ALWAYS_INLINE void add_label(const MarkedLabel& label, MarkedMetric*& metric) noexcept { - if (label.value.is_empty()) [[unlikely]] { - return; - } + PROMPP_ALWAYS_INLINE void add_label(MarkedLabel label) noexcept { + const auto& metric = metric_buffer_.back(); - auto name = label.name; - if (name.is_reserved_name()) [[unlikely]] { - name.offset = 0; - name.length = 0; + if (label.name.is_reserved_name()) [[unlikely]] { + label.name.offset = 0; + label.name.length = 0; + } else { + label.name.offset -= metric.base_offset; } + label.value.offset -= metric.base_offset; - const uint8_t sz0 = encode_size(name.offset); - const uint8_t sz1 = encode_size(name.length); + const uint8_t sz0 = encode_size(label.name.offset); + const uint8_t sz1 = encode_size(label.name.length); const uint8_t sz2 = encode_size(label.value.offset); const uint8_t sz3 = encode_size(label.value.length); @@ -346,10 +523,10 @@ class Scraper { *out++ = static_cast(layout); - *reinterpret_cast(out) = name.offset; + *reinterpret_cast(out) = label.name.offset; out += sz0 + 1; - *reinterpret_cast(out) = name.length; + *reinterpret_cast(out) = label.name.length; out += sz1 + 1; *reinterpret_cast(out) = label.value.offset; @@ -358,34 +535,22 @@ class Scraper { *reinterpret_cast(out) = label.value.length; out += sz3 + 1; - const auto offset = reinterpret_cast(metric) - this->buffer_.data(); - - this->buffer_.push_back(tmp.data(), out); - - metric = reinterpret_cast(this->buffer_.data() + offset); + bytes_buffer_.push_back(tmp.data(), out); } - PROMPP_ALWAYS_INLINE void add_sample(const MarkedSample& sample, MarkedMetric*& metric) noexcept { - const auto offset = reinterpret_cast(metric) - this->buffer_.data(); - - add_sample_internal(sample); - - metric = reinterpret_cast(this->buffer_.data() + offset); - } - - PROMPP_ALWAYS_INLINE void add_sample_internal(const MarkedSample& sample) noexcept { + PROMPP_ALWAYS_INLINE void add_sample(const MarkedSample& sample) noexcept { uint8_t marker = sample.has_ts ? 0b10000000 : 0; const double val = sample.sample.value(); auto flush = [&](uint8_t m) PROMPP_LAMBDA_INLINE { - this->buffer_.push_back(marker | m); + bytes_buffer_.push_back(marker | m); if (sample.has_ts) { append(sample.sample.timestamp()); } }; if (std::isnan(val)) [[unlikely]] { - flush(0b00000100); // staleNaN + flush(0b00000100); // NaN return; } @@ -424,10 +589,13 @@ class Scraper { } private: + BareBones::Vector metric_buffer_; + BareBones::Vector bytes_buffer_; + template PROMPP_ALWAYS_INLINE void append(const T val) noexcept { auto p = reinterpret_cast(&val); - this->buffer_.push_back(p, p + sizeof(T)); + bytes_buffer_.push_back(p, p + sizeof(T)); } static uint8_t encode_size(uint32_t v) noexcept { @@ -460,10 +628,12 @@ class Scraper { public: MetricParser(Parser& parser, MetricMarkupBuffer& markup_buffer, - MarkedMetric* metric, BareBones::Vector& labels, + uint32_t global_offset, Primitives::Timestamp timestamp) - : parser_(parser), markup_buffer_(markup_buffer), metric_(metric), labels_(labels), sample_{.sample = {timestamp, 0.0}} {} + : parser_(parser), markup_buffer_(markup_buffer), labels_(labels), sample_{.sample = {timestamp, 0.0}} { + markup_buffer_.add_metric(global_offset); + } [[nodiscard]] Error parse() noexcept { labels_.clear(); @@ -471,15 +641,8 @@ class Scraper { bool have_metric_name = false; auto& tokenizer = parser_.tokenizer(); - metric_->offset = tokenizer.token_str().data() - tokenizer.buffer().data(); - if (tokenizer.token() == Token::kMetricName) [[likely]] { - // markup_buffer_.add_label(MarkedLabel{.value = MarkedString::create(tokenizer.token_str(), tokenizer.buffer())}, metric_); - - auto string = MarkedString::create(tokenizer.token_str(), tokenizer.buffer()); - string.offset -= metric_->offset; - auto label = MarkedLabel{.value = string}; - labels_.push_back(label); + labels_.push_back(MarkedLabel{.value = MarkedString::create(tokenizer.token_str(), tokenizer.buffer())}); have_metric_name = true; tokenizer.next_non_whitespace(); @@ -501,8 +664,12 @@ class Scraper { return Error::kNoMetricName; } - // metric_->calculate_hash(tokenizer.buffer()); // sort + { + const auto it = std::remove_if(labels_.begin(), labels_.end(), [](const MarkedLabel& label) { return label.value.is_empty(); }); + labels_.erase(it, labels_.end()); + } + std::sort(labels_.begin(), labels_.end(), [buffer = tokenizer.buffer()](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { return a.name.view(buffer) < b.name.view(buffer); }); @@ -513,17 +680,17 @@ class Scraper { for (const auto& label : labels_) { hash.extend(label.name.view(tokenizer.buffer()), label.value.view(tokenizer.buffer())); } - metric_->hash = hash.hash(); + markup_buffer_.add_hash(hash.hash()); } // encode count { - markup_buffer_.add_count(labels_.size(), metric_); + markup_buffer_.add_count(labels_.size()); } // encode labels for (const auto& label : labels_) { - markup_buffer_.add_label(label, metric_); + markup_buffer_.add_label(label); } return parse_metric_suffix(); @@ -532,7 +699,6 @@ class Scraper { private: Parser& parser_; MetricMarkupBuffer& markup_buffer_; - MarkedMetric* metric_; BareBones::Vector& labels_; MarkedSample sample_; @@ -555,14 +721,12 @@ class Scraper { return error; } - // markup_buffer_.add_label(label, metric_); labels_.push_back(label); tokenizer.next(); } else { if (!have_metric_name) [[unlikely]] { - // markup_buffer_.add_label(MarkedLabel{.value = label.name}, metric_); - labels_.push_back(label); + labels_.push_back(MarkedLabel{.value = label.name}); have_metric_name = true; } else { @@ -585,7 +749,6 @@ class Scraper { if (tokenizer.token() == Token::kLabelName) [[likely]] { label_name = MarkedString::create(tokenizer.token_str(), tokenizer.buffer()); - label_name.offset -= metric_->offset; return Error::kNoError; } if (tokenizer.token() == Token::kQuotedString) { @@ -616,7 +779,6 @@ class Scraper { } string = MarkedString::create(value, tokenizer.buffer()); - string.offset -= metric_->offset; return Error::kNoError; } @@ -629,7 +791,7 @@ class Scraper { return error; } - markup_buffer_.add_sample(sample_, metric_); + markup_buffer_.add_sample(sample_); return parser_.validate_parse_sample_result(); } @@ -694,6 +856,7 @@ class Scraper { MetricMarkupBuffer metric_buffer_; MetadataMarkupBuffer metadata_buffer_; BareBones::Vector labels_; + Primitives::Timestamp default_timestamp_{}; }; using PrometheusScraper = Scraper; From 9b62e6ae67cd44b8660a2b8c50cfe8ec31c2f263 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Mon, 1 Sep 2025 12:27:51 +0300 Subject: [PATCH 06/48] read/write optimization --- pp/wal/hashdex/scraper/scraper.h | 266 ++++++++++++++++++------------- 1 file changed, 154 insertions(+), 112 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 660f13e14e..3f16adb6e6 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -16,12 +16,16 @@ #include "primitives/sample.h" +// #include "tracy/Tracy.hpp" + namespace PromPP::WAL::hashdex::scraper { template class Scraper { public: [[nodiscard]] Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { + // ZoneScopedN("Scraper::parse"); + labels_.clear(); labels_.reserve(255); default_timestamp_ = default_timestamp; @@ -181,7 +185,8 @@ class Scraper { template void read(Timeseries& ts) const { - const char* ptr = bytes_buffer_.data() + item_->data_offset; + // ZoneScopedN("Metric::read()"); + const char* ptr = reinterpret_cast(bytes_buffer_.data() + item_->data_offset); const char* base = buffer_.data() + item_->base_offset; // decode label count @@ -196,19 +201,12 @@ class Scraper { const uint8_t sz2 = (layout >> 4) & 0x3; const uint8_t sz3 = (layout >> 6) & 0x3; - auto read_val = [&](uint8_t sz) PROMPP_LAMBDA_INLINE -> uint32_t { - uint32_t v = 0; - memcpy(&v, ptr, sz + 1); - ptr += sz + 1; - return v; - }; - - uint32_t name_off = read_val(sz0); - uint32_t name_len = read_val(sz1); - uint32_t value_off = read_val(sz2); - uint32_t value_len = read_val(sz3); + const uint32_t name_off = read_val_partial(ptr, sz0); + const uint32_t name_len = read_val_partial(ptr, sz1); + const uint32_t value_off = read_val_partial(ptr, sz2); + const uint32_t value_len = read_val_partial(ptr, sz3); - if (name_off == 0 && name_len == 0) [[unlikely]] { + if (name_len == 0 && name_off == 0) [[unlikely]] { ts.label_set().append(Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); } else { ts.label_set().append(std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); @@ -233,45 +231,50 @@ class Scraper { double val; switch (type) { - case 0b00000011: { // uint32 - uint32_t tmp; - memcpy(&tmp, ptr, sizeof(tmp)); - ptr += sizeof(tmp); - val = static_cast(tmp); - break; - } case 0b00000000: // zero val = 0.0; break; + case 0b00000001: { // uint8 - uint8_t tmp = static_cast(*ptr++); - val = static_cast(tmp); + val = static_cast(*ptr++); break; } + case 0b00000010: { // uint16 - uint16_t tmp; - memcpy(&tmp, ptr, sizeof(tmp)); - ptr += sizeof(tmp); - val = static_cast(tmp); + uint16_t x = static_cast(ptr[0]) | (static_cast(ptr[1]) << 8); + ptr += 2; + val = static_cast(x); break; } - case 0b00000100: // NaN + + case 0b00000011: { // uint32 + uint32_t x; + std::memcpy(&x, ptr, sizeof(x)); + ptr += 4; + val = static_cast(x); + break; + } + + case 0b00000100: // NaN (normal) val = Prometheus::kNormalNan; break; + case 0b00001000: { // float32 - float tmp; - memcpy(&tmp, ptr, sizeof(tmp)); - ptr += sizeof(tmp); - val = static_cast(tmp); + float x; + std::memcpy(&x, ptr, sizeof(x)); + ptr += sizeof(x); + val = static_cast(x); break; } + case 0b00001001: { // double - double tmp; - memcpy(&tmp, ptr, sizeof(tmp)); - ptr += sizeof(tmp); - val = tmp; + double x; + std::memcpy(&x, ptr, sizeof(x)); + ptr += sizeof(x); + val = x; break; } + default: val = Prometheus::kStaleNan; break; @@ -287,7 +290,7 @@ class Scraper { const MarkedMetric* item_{}; Primitives::Timestamp default_timestamp_; - static uint32_t decode_varint(const char*& ptr) noexcept { + PROMPP_ALWAYS_INLINE static uint32_t decode_varint(const char*& ptr) noexcept { uint32_t b0 = static_cast(*ptr++); if ((b0 & 0x80) == 0) [[likely]] { return b0; @@ -315,6 +318,33 @@ class Scraper { v |= (b4 & 0x0F) << 28; return v; } + + PROMPP_ALWAYS_INLINE + static uint32_t read_val_partial(const char*& p, uint8_t sz) noexcept { + if (sz == 0) [[likely]] { + const uint32_t v = static_cast(p[0]); + p += 1; + return v; + } + + if (sz == 1) { + const uint32_t v = static_cast(static_cast(p[0])) | (static_cast(static_cast(p[1])) << 8); + p += 2; + return v; + } + + if (sz == 2) { + const uint32_t v = static_cast(static_cast(p[0])) | (static_cast(static_cast(p[1])) << 8) | + (static_cast(static_cast(p[2])) << 16); + p += 3; + return v; + } + + uint32_t v; + std::memcpy(&v, p, 4); + p += 4; + return v; + } }; class Metadata { @@ -465,51 +495,54 @@ class Scraper { } PROMPP_ALWAYS_INLINE void add_count(uint32_t count) noexcept { - constexpr uint32_t kVarint1Byte = 0x80; // 2^7 - constexpr uint32_t kVarint2Byte = 0x4000; // 2^14 - constexpr uint32_t kVarint3Byte = 0x200000; // 2^21 - constexpr uint32_t kVarint4Byte = 0x10000000; // 2^28 constexpr uint8_t kContinueBit = 0x80; constexpr uint8_t kValueMask = 0x7F; - std::array tmp{}; - char* out = tmp.data(); + const uint32_t offset = bytes_count(); - if (count < kVarint1Byte) { - *out++ = static_cast(count); - } else if (count < kVarint2Byte) { + if (count < (1u << 7)) [[likely]] { + bytes_buffer_.resize(bytes_buffer_.size() + 1); + char* out = bytes_buffer_.data() + offset; + *out = static_cast(count); + } else if (count < (1u << 14)) { + bytes_buffer_.resize(bytes_buffer_.size() + 2); + char* out = bytes_buffer_.data() + offset; *out++ = static_cast((count & kValueMask) | kContinueBit); - *out++ = static_cast(count >> 7); - } else if (count < kVarint3Byte) { + *out = static_cast(count >> 7); + } else if (count < (1u << 21)) { + bytes_buffer_.resize(bytes_buffer_.size() + 3); + char* out = bytes_buffer_.data() + offset; *out++ = static_cast((count & kValueMask) | kContinueBit); *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); - *out++ = static_cast(count >> 14); - } else if (count < kVarint4Byte) { + *out = static_cast(count >> 14); + } else if (count < (1u << 28)) { + bytes_buffer_.resize(bytes_buffer_.size() + 4); + char* out = bytes_buffer_.data() + offset; *out++ = static_cast((count & kValueMask) | kContinueBit); *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); *out++ = static_cast(((count >> 14) & kValueMask) | kContinueBit); - *out++ = static_cast(count >> 21); + *out = static_cast(count >> 21); } else { + bytes_buffer_.resize(bytes_buffer_.size() + 5); + char* out = bytes_buffer_.data() + offset; *out++ = static_cast((count & kValueMask) | kContinueBit); *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); *out++ = static_cast(((count >> 14) & kValueMask) | kContinueBit); *out++ = static_cast(((count >> 21) & kValueMask) | kContinueBit); - *out++ = static_cast(count >> 28); + *out = static_cast(count >> 28); } - - bytes_buffer_.push_back(tmp.data(), out); } PROMPP_ALWAYS_INLINE void add_label(MarkedLabel label) noexcept { - const auto& metric = metric_buffer_.back(); + const auto base_offset = metric_buffer_.back().base_offset; if (label.name.is_reserved_name()) [[unlikely]] { label.name.offset = 0; label.name.length = 0; } else { - label.name.offset -= metric.base_offset; + label.name.offset -= base_offset; } - label.value.offset -= metric.base_offset; + label.value.offset -= base_offset; const uint8_t sz0 = encode_size(label.name.offset); const uint8_t sz1 = encode_size(label.name.length); @@ -518,73 +551,44 @@ class Scraper { const uint8_t layout = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); - std::array tmp{}; - char* out = tmp.data(); + const uint32_t bytes_needed = sizeof(layout) + (sz0 + sz1 + sz2 + sz3) + 4; + const uint32_t offset = bytes_count(); + bytes_buffer_.resize(bytes_buffer_.size() + 17); + char* out = bytes_buffer_.data() + offset; *out++ = static_cast(layout); - *reinterpret_cast(out) = label.name.offset; + std::memcpy(out, &label.name.offset, sz0 + 1); out += sz0 + 1; - - *reinterpret_cast(out) = label.name.length; + std::memcpy(out, &label.name.length, sz1 + 1); out += sz1 + 1; - - *reinterpret_cast(out) = label.value.offset; + std::memcpy(out, &label.value.offset, sz2 + 1); out += sz2 + 1; + std::memcpy(out, &label.value.length, sz3 + 1); - *reinterpret_cast(out) = label.value.length; - out += sz3 + 1; - - bytes_buffer_.push_back(tmp.data(), out); + bytes_buffer_.resize(offset + bytes_needed); } PROMPP_ALWAYS_INLINE void add_sample(const MarkedSample& sample) noexcept { - uint8_t marker = sample.has_ts ? 0b10000000 : 0; const double val = sample.sample.value(); - - auto flush = [&](uint8_t m) PROMPP_LAMBDA_INLINE { - bytes_buffer_.push_back(marker | m); - if (sample.has_ts) { - append(sample.sample.timestamp()); - } - }; + const bool has_ts = sample.has_ts; if (std::isnan(val)) [[unlikely]] { - flush(0b00000100); // NaN + append_sample_marker(0b00000100, has_ts); // NaN + append_timestamp_if_needed(sample); return; } if (val == 0.0) [[unlikely]] { - flush(0b00000000); // zero + append_sample_marker(0b00000000, has_ts); // zero + append_timestamp_if_needed(sample); return; } - if (std::trunc(val) == val && val > 0.0) [[likely]] { // u integer - auto ival = static_cast(val); - if (ival <= std::numeric_limits::max()) [[unlikely]] { - flush(0b00000001); - append(static_cast(ival)); - return; - } - if (ival <= std::numeric_limits::max()) [[unlikely]] { - flush(0b00000010); - append(static_cast(ival)); - return; - } - if (ival <= std::numeric_limits::max()) [[likely]] { - flush(0b00000011); - append(static_cast(ival)); - return; - } - } - - float f = static_cast(val); - if (static_cast(f) == val) [[unlikely]] { - flush(0b00001000); // float32 - append(f); + if (std::trunc(val) == val && val > 0.0) [[likely]] { + append_integer_sample(sample, val); } else { - flush(0b00001001); // double - append(val); + append_floating_sample(sample, val); } } @@ -592,20 +596,57 @@ class Scraper { BareBones::Vector metric_buffer_; BareBones::Vector bytes_buffer_; - template - PROMPP_ALWAYS_INLINE void append(const T val) noexcept { + PROMPP_ALWAYS_INLINE void append_sample_marker(uint8_t type, bool has_ts) noexcept { bytes_buffer_.push_back((has_ts ? 0b10000000 : 0) | type); } + + PROMPP_ALWAYS_INLINE void append_timestamp_if_needed(const MarkedSample& sample) noexcept { + if (sample.has_ts) { + append_value(sample.sample.timestamp()); + } + } + + PROMPP_ALWAYS_INLINE void append_integer_sample(const MarkedSample& sample, double val) noexcept { + const bool has_ts = sample.has_ts; + const uint64_t uval = static_cast(val); + if (uval <= std::numeric_limits::max()) { + append_sample_marker(0b00000001, has_ts); + append_timestamp_if_needed(sample); + append_value(static_cast(uval)); + } else if (uval <= std::numeric_limits::max()) { + append_sample_marker(0b00000010, has_ts); + append_timestamp_if_needed(sample); + append_value(static_cast(uval)); + } else if (uval <= std::numeric_limits::max()) { + append_sample_marker(0b00000011, has_ts); + append_timestamp_if_needed(sample); + append_value(static_cast(uval)); + } else { + append_floating_sample(sample, val); + } + } + + PROMPP_ALWAYS_INLINE void append_floating_sample(const MarkedSample& sample, double val) noexcept { + const bool has_ts = sample.has_ts; + float f = static_cast(val); + if (static_cast(f) == val) [[unlikely]] { + append_sample_marker(0b00001000, has_ts); // float32 + append_timestamp_if_needed(sample); + append_value(f); + } else { + append_sample_marker(0b00001001, has_ts); // double + append_timestamp_if_needed(sample); + append_value(val); + } + } + + template + PROMPP_ALWAYS_INLINE void append_value(T val) noexcept { auto p = reinterpret_cast(&val); bytes_buffer_.push_back(p, p + sizeof(T)); } - static uint8_t encode_size(uint32_t v) noexcept { - if (v <= 0xFF) - return 0; - if (v <= 0xFFFF) - return 1; - if (v <= 0xFFFFFF) - return 2; - return 3; + PROMPP_ALWAYS_INLINE static uint8_t encode_size(uint32_t v) noexcept { + const uint32_t msb = (v == 0 ? 0 : 31 - std::countl_zero(v)); + return msb >> 3; } }; @@ -636,6 +677,7 @@ class Scraper { } [[nodiscard]] Error parse() noexcept { + // ZoneScopedN("MetricParser::parse"); labels_.clear(); bool have_metric_name = false; From aca48508f00cba2e0ddc95a6da3d534463d0d298 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Mon, 1 Sep 2025 17:21:39 +0300 Subject: [PATCH 07/48] sample optimization --- pp/wal/hashdex/scraper/scraper.h | 116 ++++++++++++------------------- 1 file changed, 46 insertions(+), 70 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 3f16adb6e6..ebc3075b76 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -2,7 +2,6 @@ #include -#include #include #include "bare_bones/algorithm.h" @@ -16,15 +15,12 @@ #include "primitives/sample.h" -// #include "tracy/Tracy.hpp" - namespace PromPP::WAL::hashdex::scraper { template class Scraper { public: [[nodiscard]] Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { - // ZoneScopedN("Scraper::parse"); labels_.clear(); labels_.reserve(255); default_timestamp_ = default_timestamp; @@ -185,7 +181,6 @@ class Scraper { template void read(Timeseries& ts) const { - // ZoneScopedN("Metric::read()"); const char* ptr = reinterpret_cast(bytes_buffer_.data() + item_->data_offset); const char* base = buffer_.data() + item_->base_offset; @@ -221,13 +216,6 @@ class Scraper { Primitives::Sample sample{}; sample.timestamp() = default_timestamp_; - if (has_ts) [[unlikely]] { - int64_t ts_val; - memcpy(&ts_val, ptr, sizeof(ts_val)); - ptr += sizeof(ts_val); - sample.timestamp() = ts_val; - } - double val; switch (type) { @@ -280,6 +268,12 @@ class Scraper { break; } + if (has_ts) [[unlikely]] { + Primitives::Timestamp ts_val; + memcpy(&ts_val, ptr, sizeof(ts_val)); + sample.timestamp() = ts_val; + } + sample.value() = val; ts.samples().emplace_back(sample); } @@ -573,75 +567,58 @@ class Scraper { const double val = sample.sample.value(); const bool has_ts = sample.has_ts; + constexpr uint32_t max_sample_bytes = 1 + sizeof(sample.sample); // marker + value + timestamp + const uint32_t offset = bytes_count(); + bytes_buffer_.resize(offset + max_sample_bytes); + char* out = bytes_buffer_.data() + offset; + char* start = out; + if (std::isnan(val)) [[unlikely]] { - append_sample_marker(0b00000100, has_ts); // NaN - append_timestamp_if_needed(sample); - return; + *out++ = static_cast((has_ts ? 0b10000000 : 0) | 0b00000100); // NaN + } else if (val == 0.0) [[unlikely]] { + *out++ = static_cast((has_ts ? 0b10000000 : 0) | 0b00000000); // Zero + } else if (std::trunc(val) == val && val > 0.0) [[likely]] { + const uint64_t uval = static_cast(val); + if (uval <= std::numeric_limits::max()) { + const auto v = static_cast(uval); + out = write_marker_and_value(out, 0b00000001, has_ts, v); + } else if (uval <= std::numeric_limits::max()) { + const auto v = static_cast(uval); + out = write_marker_and_value(out, 0b00000010, has_ts, v); + } else if (uval <= std::numeric_limits::max()) { + const auto v = static_cast(uval); + out = write_marker_and_value(out, 0b00000011, has_ts, v); + } else { + out = write_marker_and_value(out, 0b00001001, has_ts, val); // double + } + } else { + float f = static_cast(val); + if (static_cast(f) == val) [[unlikely]] { + out = write_marker_and_value(out, 0b00001000, has_ts, f); // float + } else { + out = write_marker_and_value(out, 0b00001001, has_ts, val); // double + } } - if (val == 0.0) [[unlikely]] { - append_sample_marker(0b00000000, has_ts); // zero - append_timestamp_if_needed(sample); - return; + if (has_ts) { + const auto ts = sample.sample.timestamp(); + std::memcpy(out, &ts, sizeof(ts)); + out += sizeof(ts); } - if (std::trunc(val) == val && val > 0.0) [[likely]] { - append_integer_sample(sample, val); - } else { - append_floating_sample(sample, val); - } + const uint32_t written = static_cast(out - start); + bytes_buffer_.resize(offset + written); } private: BareBones::Vector metric_buffer_; BareBones::Vector bytes_buffer_; - PROMPP_ALWAYS_INLINE void append_sample_marker(uint8_t type, bool has_ts) noexcept { bytes_buffer_.push_back((has_ts ? 0b10000000 : 0) | type); } - - PROMPP_ALWAYS_INLINE void append_timestamp_if_needed(const MarkedSample& sample) noexcept { - if (sample.has_ts) { - append_value(sample.sample.timestamp()); - } - } - - PROMPP_ALWAYS_INLINE void append_integer_sample(const MarkedSample& sample, double val) noexcept { - const bool has_ts = sample.has_ts; - const uint64_t uval = static_cast(val); - if (uval <= std::numeric_limits::max()) { - append_sample_marker(0b00000001, has_ts); - append_timestamp_if_needed(sample); - append_value(static_cast(uval)); - } else if (uval <= std::numeric_limits::max()) { - append_sample_marker(0b00000010, has_ts); - append_timestamp_if_needed(sample); - append_value(static_cast(uval)); - } else if (uval <= std::numeric_limits::max()) { - append_sample_marker(0b00000011, has_ts); - append_timestamp_if_needed(sample); - append_value(static_cast(uval)); - } else { - append_floating_sample(sample, val); - } - } - - PROMPP_ALWAYS_INLINE void append_floating_sample(const MarkedSample& sample, double val) noexcept { - const bool has_ts = sample.has_ts; - float f = static_cast(val); - if (static_cast(f) == val) [[unlikely]] { - append_sample_marker(0b00001000, has_ts); // float32 - append_timestamp_if_needed(sample); - append_value(f); - } else { - append_sample_marker(0b00001001, has_ts); // double - append_timestamp_if_needed(sample); - append_value(val); - } - } - template - PROMPP_ALWAYS_INLINE void append_value(T val) noexcept { - auto p = reinterpret_cast(&val); - bytes_buffer_.push_back(p, p + sizeof(T)); + PROMPP_ALWAYS_INLINE static char* write_marker_and_value(char* out, uint8_t marker, bool has_ts, const T& val) noexcept { + *out++ = static_cast((has_ts ? 0b10000000 : 0) | marker); + std::memcpy(out, &val, sizeof(T)); + return out + sizeof(T); } PROMPP_ALWAYS_INLINE static uint8_t encode_size(uint32_t v) noexcept { @@ -677,7 +654,6 @@ class Scraper { } [[nodiscard]] Error parse() noexcept { - // ZoneScopedN("MetricParser::parse"); labels_.clear(); bool have_metric_name = false; From 26d0af47911d1c6c44a02f6597491a8035182eca Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Mon, 1 Sep 2025 17:29:31 +0300 Subject: [PATCH 08/48] scraper read benchmark 2 shards --- pp/wal/benchmarks/scraper_benchmark.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pp/wal/benchmarks/scraper_benchmark.cpp b/pp/wal/benchmarks/scraper_benchmark.cpp index 152079318b..45385e2faf 100644 --- a/pp/wal/benchmarks/scraper_benchmark.cpp +++ b/pp/wal/benchmarks/scraper_benchmark.cpp @@ -76,7 +76,9 @@ void BenchmarkScraperRead(benchmark::State& state) { for ([[maybe_unused]] auto _ : state) { PromPP::Primitives::TimeseriesSemiview ts; for (auto& metric : scraper.metrics()) { - metric.read(ts); + if (metric.hash() % 2 == 0) { + metric.read(ts); + } } } } From 9f623c78bd6a0cd66d31643e13a939eb2fc687fd Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Mon, 1 Sep 2025 18:53:24 +0300 Subject: [PATCH 09/48] restore MarkupBuffer base class --- pp/wal/hashdex/scraper/scraper.h | 155 ++++++++++++------------------- 1 file changed, 57 insertions(+), 98 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index ebc3075b76..58a68043cf 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -169,10 +169,16 @@ class Scraper { public: class Metric { public: - using MarkedItem = MarkedMetric; + using MarkedT = MarkedMetric; - Metric(std::string_view buffer, const BareBones::Vector& bytes_buffer, const MarkedMetric* item, Primitives::Timestamp default_timestamp) - : buffer_(buffer), bytes_buffer_(bytes_buffer), item_(item), default_timestamp_(default_timestamp) {} + struct Context { + std::string_view buffer; + const BareBones::Vector& bytes_buffer; + Primitives::Timestamp default_timestamp; + }; + + Metric(const Context& ctx, const MarkedMetric* item) + : buffer_(ctx.buffer), bytes_buffer_(ctx.bytes_buffer), item_(item), default_timestamp_(ctx.default_timestamp) {} [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetric* item() const noexcept { return item_; } PROMPP_ALWAYS_INLINE void set_item(const MarkedMetric* item) noexcept { item_ = item; } @@ -343,9 +349,13 @@ class Scraper { class Metadata { public: - using MarkedItem = MarkedMetadata; + using MarkedT = MarkedMetadata; + + struct Context { + std::string_view buffer; + }; - explicit Metadata(std::string_view buffer, const MarkedMetadata* item) : buffer_(buffer), item_(item) {} + Metadata(const Context& ctx, const MarkedMetadata* item) : buffer_(ctx.buffer), item_(item) {} [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetadata* item() const noexcept { return item_; } PROMPP_ALWAYS_INLINE void set_item(const MarkedMetadata* item) noexcept { item_ = item; } @@ -359,133 +369,78 @@ class Scraper { const MarkedMetadata* item_{}; }; - private: - template + template class MarkupBuffer { public: + using MarkedT = typename T::MarkedT; + using Context = typename T::Context; + class IteratorSentinel {}; class Iterator { public: using iterator_category = std::forward_iterator_tag; - using value_type = Item; + using value_type = T; using difference_type = ptrdiff_t; using pointer = value_type*; using reference = value_type&; - using MarkedItem = typename Item::MarkedItem; - Iterator(std::string_view buffer, const MarkupBuffer* markup_buffer) - : item_(buffer, reinterpret_cast(markup_buffer->buffer().data())), items_count_(markup_buffer->items_count()) {} + Iterator(const Context& ctx, const MarkedT* ptr, uint32_t items_count) : item_(ctx, ptr), ptr_(ptr), items_count_(items_count), ctx_(ctx) {} [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type& operator*() const noexcept { return item_; } [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type* operator->() const noexcept { return &item_; } PROMPP_ALWAYS_INLINE Iterator& operator++() noexcept { - item_.set_item(reinterpret_cast(reinterpret_cast(item_.item()) + sizeof(*item_.item()))); + item_.set_item(++ptr_); --items_count_; return *this; } PROMPP_ALWAYS_INLINE Iterator operator++(int) noexcept { - const auto it = *this; - ++*this; - return it; + auto tmp = *this; + ++(*this); + return tmp; } PROMPP_ALWAYS_INLINE bool operator==(const IteratorSentinel&) const noexcept { return items_count_ == 0; } private: - Item item_; + T item_; + const MarkedT* ptr_; uint32_t items_count_; + Context ctx_; }; - [[nodiscard]] PROMPP_ALWAYS_INLINE const BareBones::Vector& buffer() const noexcept { return buffer_; } - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t items_count() const noexcept { return items_count_; } - - PROMPP_ALWAYS_INLINE void remove_item() noexcept { --items_count_; } - - PROMPP_ALWAYS_INLINE void initialize(size_t reserve) noexcept { - buffer_.clear(); - buffer_.reserve(reserve); - items_count_ = 0; - } - - [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer) const noexcept { return {buffer, this}; } - [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } - + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t items_count() const noexcept { return buffer_.size(); } [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return buffer_.allocated_memory(); } protected: - BareBones::Vector buffer_; - uint32_t items_count_{}; + BareBones::Vector buffer_; }; - class MetricMarkupBuffer { + class MetricMarkupBuffer : public MarkupBuffer { public: - class IteratorSentinel {}; - - class Iterator { - public: - using iterator_category = std::forward_iterator_tag; - using value_type = Metric; - using difference_type = ptrdiff_t; - using pointer = value_type*; - using reference = value_type&; - - Iterator(std::string_view buffer, - const BareBones::Vector& bytes_buffer, - const MarkedMetric* ptr, - uint32_t items_count, - Primitives::Timestamp default_timestamp) - : item_(buffer, bytes_buffer, ptr, default_timestamp), ptr_(ptr), buffer_(buffer), bytes_buffer_(bytes_buffer), items_count_(items_count) {} - - [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type& operator*() const noexcept { return item_; } - [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type* operator->() const noexcept { return &item_; } + using Base = MarkupBuffer; + using Iterator = typename Base::Iterator; + using IteratorSentinel = typename Base::IteratorSentinel; - PROMPP_ALWAYS_INLINE Iterator& operator++() noexcept { - item_.set_item(++ptr_); - --items_count_; - return *this; - } - - PROMPP_ALWAYS_INLINE Iterator operator++(int) noexcept { - auto tmp = *this; - ++(*this); - return tmp; - } - - [[nodiscard]] PROMPP_ALWAYS_INLINE bool operator==(const IteratorSentinel&) const noexcept { return items_count_ == 0; } - - private: - Metric item_; - const MarkedMetric* ptr_{}; - std::string_view buffer_; - const BareBones::Vector& bytes_buffer_; - uint32_t items_count_{}; - }; - - [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer, Primitives::Timestamp default_timestamp) const noexcept { - return {buffer, bytes_buffer_, metric_buffer_.data(), metric_buffer_.size(), default_timestamp}; + [[nodiscard]] Iterator begin(std::string_view buffer, Primitives::Timestamp default_ts) const noexcept { + return {typename Base::Context{buffer, bytes_buffer_, default_ts}, this->buffer_.data(), this->items_count()}; } + [[nodiscard]] static IteratorSentinel end() noexcept { return {}; } - [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } - - [[nodiscard]] PROMPP_ALWAYS_INLINE const BareBones::Vector& metric_buffer() const noexcept { return metric_buffer_; } - [[nodiscard]] PROMPP_ALWAYS_INLINE const BareBones::Vector& bytes_buffer() const noexcept { return bytes_buffer_; } - - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t items_count() const noexcept { return metric_buffer_.size(); } [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t bytes_count() const noexcept { return bytes_buffer_.size(); } - [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return metric_buffer_.allocated_memory() + bytes_buffer_.allocated_memory(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return this->buffer_.allocated_memory() + bytes_buffer_.allocated_memory(); } PROMPP_ALWAYS_INLINE void remove_item() noexcept { - bytes_buffer_.resize(metric_buffer_.back().data_offset); - metric_buffer_.pop_back(); + bytes_buffer_.resize(this->buffer_.back().data_offset); + this->buffer_.pop_back(); } - PROMPP_ALWAYS_INLINE void add_hash(uint64_t hash) noexcept { metric_buffer_.back().hash = hash; } + PROMPP_ALWAYS_INLINE void add_hash(uint64_t hash) noexcept { this->buffer_.back().hash = hash; } PROMPP_ALWAYS_INLINE void add_metric(uint32_t global_offset) noexcept { - metric_buffer_.push_back(MarkedMetric{.base_offset = global_offset, .data_offset = bytes_count()}); + this->buffer_.push_back(MarkedMetric{.base_offset = global_offset, .data_offset = bytes_count()}); } PROMPP_ALWAYS_INLINE void add_count(uint32_t count) noexcept { @@ -528,7 +483,7 @@ class Scraper { } PROMPP_ALWAYS_INLINE void add_label(MarkedLabel label) noexcept { - const auto base_offset = metric_buffer_.back().base_offset; + const auto base_offset = this->buffer_.back().base_offset; if (label.name.is_reserved_name()) [[unlikely]] { label.name.offset = 0; @@ -611,7 +566,6 @@ class Scraper { } private: - BareBones::Vector metric_buffer_; BareBones::Vector bytes_buffer_; template @@ -629,16 +583,21 @@ class Scraper { class MetadataMarkupBuffer : public MarkupBuffer { public: + using Base = MarkupBuffer; + using Iterator = typename Base::Iterator; + using IteratorSentinel = typename Base::IteratorSentinel; + + [[nodiscard]] Iterator begin(std::string_view buffer) const noexcept { return {typename Base::Context{buffer}, this->buffer_.data(), this->items_count()}; } + [[nodiscard]] static IteratorSentinel end() noexcept { return {}; } + PROMPP_ALWAYS_INLINE void add(MarkedString metric_name, MarkedString text, Prometheus::MetadataType type) noexcept { - ++this->items_count_; - - const auto offset = this->buffer_.size(); - this->buffer_.resize(offset + sizeof(MarkedMetadata)); - new (reinterpret_cast(this->buffer_.data() + offset)) MarkedMetadata{ - .metric_name = metric_name, - .text = text, - .type = type, - }; + this->buffer_.emplace_back(metric_name, text, type); + } + + void remove_item() noexcept { this->buffer_.pop_back(); } + void initialize(size_t reserve) noexcept { + this->buffer_.clear(); + this->buffer_.reserve(reserve); } }; From 2036d7eaa7271779c08a938055c08270e1a207a4 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Mon, 1 Sep 2025 19:18:56 +0300 Subject: [PATCH 10/48] change MetricParser into parse_metric method --- pp/wal/hashdex/scraper/scraper.h | 330 +++++++++++++++---------------- 1 file changed, 155 insertions(+), 175 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 58a68043cf..8f84644da5 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -20,8 +20,7 @@ namespace PromPP::WAL::hashdex::scraper { template class Scraper { public: - [[nodiscard]] Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { - labels_.clear(); + [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { labels_.reserve(255); default_timestamp_ = default_timestamp; @@ -52,10 +51,7 @@ class Scraper { case Token::kMetricName: case Token::kBraceOpen: { - if (const auto error = MetricParser{parser_, metric_buffer_, labels_, static_cast(tokenizer.token_str().data() - tokenizer.buffer().data()), - default_timestamp} - .parse(); - error != Error::kNoError) [[unlikely]] { + if (const auto error = parse_metric(); error != Error::kNoError) [[unlikely]] { metric_buffer_.remove_item(); return error; } @@ -601,232 +597,216 @@ class Scraper { } }; - class MetricParser { - public: - MetricParser(Parser& parser, - MetricMarkupBuffer& markup_buffer, - BareBones::Vector& labels, - uint32_t global_offset, - Primitives::Timestamp timestamp) - : parser_(parser), markup_buffer_(markup_buffer), labels_(labels), sample_{.sample = {timestamp, 0.0}} { - markup_buffer_.add_metric(global_offset); - } - - [[nodiscard]] Error parse() noexcept { - labels_.clear(); - - bool have_metric_name = false; - auto& tokenizer = parser_.tokenizer(); - - if (tokenizer.token() == Token::kMetricName) [[likely]] { - labels_.push_back(MarkedLabel{.value = MarkedString::create(tokenizer.token_str(), tokenizer.buffer())}); - - have_metric_name = true; - tokenizer.next_non_whitespace(); - } else if (tokenizer.token() == Token::kWhitespace) [[likely]] { - tokenizer.next(); - } - - if (tokenizer.token() == Token::kBraceOpen) [[likely]] { - if (const auto error = tokenize_label_set(have_metric_name); error != Error::kNoError) { - return error; - } - - tokenizer.next_non_whitespace(); - } else if (!parser_.is_value_token()) [[unlikely]] { - return Error::kUnexpectedToken; - } - - if (!have_metric_name) [[unlikely]] { - return Error::kNoMetricName; - } - - // sort - { - const auto it = std::remove_if(labels_.begin(), labels_.end(), [](const MarkedLabel& label) { return label.value.is_empty(); }); - labels_.erase(it, labels_.end()); + [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_metadata() { + static constexpr auto get_metadata_type = [](Token token) PROMPP_LAMBDA_INLINE { + if (token == Token::kHelp) { + return Prometheus::MetadataType::kHelp; } - - std::sort(labels_.begin(), labels_.end(), [buffer = tokenizer.buffer()](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { - return a.name.view(buffer) < b.name.view(buffer); - }); - - // hash - { - BareBones::XXHash hash; - for (const auto& label : labels_) { - hash.extend(label.name.view(tokenizer.buffer()), label.value.view(tokenizer.buffer())); - } - markup_buffer_.add_hash(hash.hash()); + if (token == Token::kType) { + return Prometheus::MetadataType::kType; } - // encode count - { - markup_buffer_.add_count(labels_.size()); - } + return Prometheus::MetadataType::kUnit; + }; - // encode labels - for (const auto& label : labels_) { - markup_buffer_.add_label(label); - } + auto& tokenizer = parser_.tokenizer(); + const auto type = tokenizer.token(); - return parse_metric_suffix(); + if (tokenizer.next_non_whitespace() != Token::kMetricName) [[unlikely]] { + return Error::kUnexpectedToken; } - private: - Parser& parser_; - MetricMarkupBuffer& markup_buffer_; - BareBones::Vector& labels_; - MarkedSample sample_; - - [[nodiscard]] Error tokenize_label_set(bool& have_metric_name) noexcept { - auto& tokenizer = parser_.tokenizer(); - tokenizer.next_non_whitespace(); + auto metric_name = tokenizer.token_str(); + Prometheus::textparse::unquote(metric_name); - while (tokenizer.token() != Token::kBraceClose) { - MarkedLabel label; - if (const auto error = get_label_name(label.name); error != Error::kNoError) [[unlikely]] { - return error; - } + if (tokenizer.next_non_whitespace() != Token::kText) [[unlikely]] { + return Error::kUnexpectedToken; + } - if (tokenizer.next_non_whitespace() == Token::kEqual) [[likely]] { - if (tokenizer.next_non_whitespace() != Token::kLabelValue) [[unlikely]] { - return Error::kUnexpectedToken; - } + const auto text = tokenizer.token_str(); + if (const auto token = tokenizer.next_non_whitespace(); !BareBones::is_in(token, Token::kLinebreak, Token::kEOF)) [[unlikely]] { + return Error::kUnexpectedToken; + } - if (const auto error = get_quoted_value(label.value); error != Error::kNoError) [[unlikely]] { - return error; - } + if (type == Token::kHelp && !simdutf::validate_utf8(text.data(), text.size())) [[unlikely]] { + return Error::kInvalidUtf8; + } - labels_.push_back(label); + const auto buffer = tokenizer.buffer(); + metadata_buffer_.add(MarkedString::create(metric_name, buffer), MarkedString::create(text, buffer), get_metadata_type(type)); + return Error::kNoError; + } - tokenizer.next(); - } else { - if (!have_metric_name) [[unlikely]] { - labels_.push_back(MarkedLabel{.value = label.name}); + [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_metric() { + labels_.clear(); - have_metric_name = true; - } else { - return Error::kUnexpectedToken; - } - } + bool have_metric_name = false; + auto& tokenizer = parser_.tokenizer(); - if (tokenizer.token() != Token::kComma && tokenizer.token() != Token::kWhitespace) { - break; - } + metric_buffer_.add_metric(tokenizer.token_str().data() - tokenizer.buffer().data()); - tokenizer.next_non_whitespace(); - } + if (tokenizer.token() == Token::kMetricName) [[likely]] { + labels_.push_back(MarkedLabel{.value = MarkedString::create(tokenizer.token_str(), tokenizer.buffer())}); - return tokenizer.token() == Token::kBraceClose ? Error::kNoError : Error::kUnexpectedToken; + have_metric_name = true; + tokenizer.next_non_whitespace(); + } else if (tokenizer.token() == Token::kWhitespace) [[likely]] { + tokenizer.next(); } - [[nodiscard]] Error get_label_name(MarkedString& label_name) const noexcept { - auto& tokenizer = parser_.tokenizer(); - - if (tokenizer.token() == Token::kLabelName) [[likely]] { - label_name = MarkedString::create(tokenizer.token_str(), tokenizer.buffer()); - return Error::kNoError; - } - if (tokenizer.token() == Token::kQuotedString) { - return get_quoted_value(label_name); + if (tokenizer.token() == Token::kBraceOpen) [[likely]] { + if (const auto error = tokenize_label_set(have_metric_name); error != Error::kNoError) { + return error; } + tokenizer.next_non_whitespace(); + } else if (!parser_.is_value_token()) [[unlikely]] { return Error::kUnexpectedToken; } - [[nodiscard]] Error get_quoted_value(MarkedString& string) const noexcept { - auto& tokenizer = parser_.tokenizer(); - - auto value = tokenizer.token_str(); - Prometheus::textparse::unquote(value); + if (!have_metric_name) [[unlikely]] { + return Error::kNoMetricName; + } - auto copy_to = const_cast(value.data()); - Prometheus::textparse::unescape_label_value(value, [©_to](const std::string_view& piece_of_string) { - if (copy_to != piece_of_string.data()) [[unlikely]] { - memmove(copy_to, piece_of_string.data(), piece_of_string.size()); - } + // sort + { + const auto it = std::remove_if(labels_.begin(), labels_.end(), [](const MarkedLabel& label) { return label.value.is_empty(); }); + labels_.erase(it, labels_.end()); + } - copy_to += piece_of_string.size(); - }); - value.remove_suffix(value.size() - (copy_to - value.data())); + std::sort(labels_.begin(), labels_.end(), [buffer = tokenizer.buffer()](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { + return a.name.view(buffer) < b.name.view(buffer); + }); - if (!simdutf::validate_utf8(value.data(), value.size())) [[unlikely]] { - return Error::kInvalidUtf8; + // hash + { + BareBones::XXHash hash; + for (const auto& label : labels_) { + hash.extend(label.name.view(tokenizer.buffer()), label.value.view(tokenizer.buffer())); } + metric_buffer_.add_hash(hash.hash()); + } - string = MarkedString::create(value, tokenizer.buffer()); - return Error::kNoError; + // encode count + { + metric_buffer_.add_count(labels_.size()); } - [[nodiscard]] Error parse_metric_suffix() noexcept { - if (!parser_.is_value_token()) [[unlikely]] { - return Error::kUnexpectedToken; - } + // encode labels + for (const auto& label : labels_) { + metric_buffer_.add_label(label); + } - if (const auto error = parse_sample(); error != Error::kNoError) { + return parse_metric_suffix(); + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE Error tokenize_label_set(bool& have_metric_name) noexcept { + auto& tokenizer = parser_.tokenizer(); + tokenizer.next_non_whitespace(); + + while (tokenizer.token() != Token::kBraceClose) { + MarkedLabel label; + if (const auto error = get_label_name(label.name); error != Error::kNoError) [[unlikely]] { return error; } - markup_buffer_.add_sample(sample_); + if (tokenizer.next_non_whitespace() == Token::kEqual) [[likely]] { + if (tokenizer.next_non_whitespace() != Token::kLabelValue) [[unlikely]] { + return Error::kUnexpectedToken; + } - return parser_.validate_parse_sample_result(); - } + if (const auto error = get_quoted_value(label.value); error != Error::kNoError) [[unlikely]] { + return error; + } - [[nodiscard]] Error parse_sample() noexcept { - auto& tokenizer = parser_.tokenizer(); + labels_.push_back(label); - if (!parse_numeric_value(tokenizer.token_str(), sample_.sample.value())) [[unlikely]] { - return Error::kInvalidValue; + tokenizer.next(); + } else { + if (!have_metric_name) [[unlikely]] { + labels_.push_back(MarkedLabel{.value = label.name}); + + have_metric_name = true; + } else { + return Error::kUnexpectedToken; + } } - if (std::isnan(sample_.sample.value())) [[unlikely]] { - sample_.sample.value() = Prometheus::kNormalNan; + + if (tokenizer.token() != Token::kComma && tokenizer.token() != Token::kWhitespace) { + break; } tokenizer.next_non_whitespace(); - - return parser_.parse_timestamp(sample_.sample.timestamp(), sample_.has_ts); } - }; - [[nodiscard]] Error parse_metadata() { - static constexpr auto get_metadata_type = [](Token token) PROMPP_LAMBDA_INLINE { - if (token == Token::kHelp) { - return Prometheus::MetadataType::kHelp; - } - if (token == Token::kType) { - return Prometheus::MetadataType::kType; - } + return tokenizer.token() == Token::kBraceClose ? Error::kNoError : Error::kUnexpectedToken; + } - return Prometheus::MetadataType::kUnit; - }; + [[nodiscard]] PROMPP_ALWAYS_INLINE Error get_label_name(MarkedString& label_name) const noexcept { + auto& tokenizer = parser_.tokenizer(); + if (tokenizer.token() == Token::kLabelName) [[likely]] { + label_name = MarkedString::create(tokenizer.token_str(), tokenizer.buffer()); + return Error::kNoError; + } + if (tokenizer.token() == Token::kQuotedString) { + return get_quoted_value(label_name); + } + + return Error::kUnexpectedToken; + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE Error get_quoted_value(MarkedString& string) const noexcept { auto& tokenizer = parser_.tokenizer(); - const auto type = tokenizer.token(); - if (tokenizer.next_non_whitespace() != Token::kMetricName) [[unlikely]] { - return Error::kUnexpectedToken; + auto value = tokenizer.token_str(); + Prometheus::textparse::unquote(value); + + auto copy_to = const_cast(value.data()); + Prometheus::textparse::unescape_label_value(value, [©_to](const std::string_view& piece_of_string) { + if (copy_to != piece_of_string.data()) [[unlikely]] { + memmove(copy_to, piece_of_string.data(), piece_of_string.size()); + } + + copy_to += piece_of_string.size(); + }); + value.remove_suffix(value.size() - (copy_to - value.data())); + + if (!simdutf::validate_utf8(value.data(), value.size())) [[unlikely]] { + return Error::kInvalidUtf8; } - auto metric_name = tokenizer.token_str(); - Prometheus::textparse::unquote(metric_name); + string = MarkedString::create(value, tokenizer.buffer()); + return Error::kNoError; + } - if (tokenizer.next_non_whitespace() != Token::kText) [[unlikely]] { + [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_metric_suffix() noexcept { + if (!parser_.is_value_token()) [[unlikely]] { return Error::kUnexpectedToken; } - const auto text = tokenizer.token_str(); - if (const auto token = tokenizer.next_non_whitespace(); !BareBones::is_in(token, Token::kLinebreak, Token::kEOF)) [[unlikely]] { - return Error::kUnexpectedToken; + MarkedSample sample{.sample = {default_timestamp_, {}}}; + if (const auto error = parse_sample(sample); error != Error::kNoError) { + return error; } + metric_buffer_.add_sample(sample); - if (type == Token::kHelp && !simdutf::validate_utf8(text.data(), text.size())) [[unlikely]] { - return Error::kInvalidUtf8; + return parser_.validate_parse_sample_result(); + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_sample(MarkedSample& sample) noexcept { + auto& tokenizer = parser_.tokenizer(); + + if (!parse_numeric_value(tokenizer.token_str(), sample.sample.value())) [[unlikely]] { + return Error::kInvalidValue; + } + if (std::isnan(sample.sample.value())) [[unlikely]] { + sample.sample.value() = Prometheus::kNormalNan; } - const auto buffer = tokenizer.buffer(); - metadata_buffer_.add(MarkedString::create(metric_name, buffer), MarkedString::create(text, buffer), get_metadata_type(type)); - return Error::kNoError; + tokenizer.next_non_whitespace(); + + return parser_.parse_timestamp(sample.sample.timestamp(), sample.has_ts); } Parser parser_; From 1113d04b83901275229439cb5e89773ce456670c Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Mon, 1 Sep 2025 19:37:46 +0300 Subject: [PATCH 11/48] parse_metric refactoring --- pp/wal/hashdex/scraper/scraper.h | 61 +++++++++++++++++--------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 8f84644da5..ba1cd3d6d1 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -435,6 +435,7 @@ class Scraper { } PROMPP_ALWAYS_INLINE void add_hash(uint64_t hash) noexcept { this->buffer_.back().hash = hash; } + PROMPP_ALWAYS_INLINE void add_metric(uint32_t global_offset) noexcept { this->buffer_.push_back(MarkedMetric{.base_offset = global_offset, .data_offset = bytes_count()}); } @@ -518,7 +519,7 @@ class Scraper { const double val = sample.sample.value(); const bool has_ts = sample.has_ts; - constexpr uint32_t max_sample_bytes = 1 + sizeof(sample.sample); // marker + value + timestamp + constexpr uint32_t max_sample_bytes = 1 + sizeof(sample.sample); const uint32_t offset = bytes_count(); bytes_buffer_.resize(offset + max_sample_bytes); char* out = bytes_buffer_.data() + offset; @@ -668,34 +669,7 @@ class Scraper { return Error::kNoMetricName; } - // sort - { - const auto it = std::remove_if(labels_.begin(), labels_.end(), [](const MarkedLabel& label) { return label.value.is_empty(); }); - labels_.erase(it, labels_.end()); - } - - std::sort(labels_.begin(), labels_.end(), [buffer = tokenizer.buffer()](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { - return a.name.view(buffer) < b.name.view(buffer); - }); - - // hash - { - BareBones::XXHash hash; - for (const auto& label : labels_) { - hash.extend(label.name.view(tokenizer.buffer()), label.value.view(tokenizer.buffer())); - } - metric_buffer_.add_hash(hash.hash()); - } - - // encode count - { - metric_buffer_.add_count(labels_.size()); - } - - // encode labels - for (const auto& label : labels_) { - metric_buffer_.add_label(label); - } + process_labels_buffer(); return parse_metric_suffix(); } @@ -809,6 +783,35 @@ class Scraper { return parser_.parse_timestamp(sample.sample.timestamp(), sample.has_ts); } + PROMPP_ALWAYS_INLINE void process_labels_buffer() noexcept { + sort_and_filter_labels(); + append_labels_hash(); + + metric_buffer_.add_count(labels_.size()); + + for (const auto& label : labels_) { + metric_buffer_.add_label(label); + } + } + + PROMPP_ALWAYS_INLINE void sort_and_filter_labels() noexcept { + const auto it = std::remove_if(labels_.begin(), labels_.end(), [](const MarkedLabel& label) { return label.value.is_empty(); }); + labels_.erase(it, labels_.end()); + + std::sort(labels_.begin(), labels_.end(), [buffer = parser_.tokenizer().buffer()](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { + return a.name.view(buffer) < b.name.view(buffer); + }); + } + + PROMPP_ALWAYS_INLINE void append_labels_hash() noexcept { + const auto& tokenizer = parser_.tokenizer(); + BareBones::XXHash hash; + for (const auto& label : labels_) { + hash.extend(label.name.view(tokenizer.buffer()), label.value.view(tokenizer.buffer())); + } + metric_buffer_.add_hash(hash.hash()); + } + Parser parser_; MetricMarkupBuffer metric_buffer_; MetadataMarkupBuffer metadata_buffer_; From 2455a51e947f9bbf075e9f416f62cd1d96196f23 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Tue, 2 Sep 2025 11:23:10 +0300 Subject: [PATCH 12/48] benchmark update --- pp/wal/benchmarks/scraper_benchmark.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pp/wal/benchmarks/scraper_benchmark.cpp b/pp/wal/benchmarks/scraper_benchmark.cpp index 45385e2faf..aa97fc6e37 100644 --- a/pp/wal/benchmarks/scraper_benchmark.cpp +++ b/pp/wal/benchmarks/scraper_benchmark.cpp @@ -73,11 +73,13 @@ void BenchmarkScraperRead(benchmark::State& state) { PrometheusScraper scraper; std::ignore = scraper.parse(str, 0); + PromPP::Primitives::TimeseriesSemiview ts_buf; + for ([[maybe_unused]] auto _ : state) { - PromPP::Primitives::TimeseriesSemiview ts; for (auto& metric : scraper.metrics()) { if (metric.hash() % 2 == 0) { - metric.read(ts); + ts_buf.clear(); + metric.read(ts_buf); } } } From 9457484c5aeca3dfdecf677bb5f7326ecac04bc7 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 3 Sep 2025 09:37:50 +0300 Subject: [PATCH 13/48] allocated_memory fix --- pp/wal/hashdex/scraper/scraper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index ba1cd3d6d1..eda2366f18 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -106,7 +106,7 @@ class Scraper { [[nodiscard]] PROMPP_ALWAYS_INLINE Primitives::Timestamp default_timestamp() const noexcept { return default_timestamp_; } [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { - return metric_buffer_.allocated_memory() + metadata_buffer_.allocated_memory(); + return metric_buffer_.allocated_memory() + metadata_buffer_.allocated_memory() + labels_.allocated_memory(); } private: From 00c833eb55bdbfdc2131aa946e215a9b69661240 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 3 Sep 2025 09:44:54 +0300 Subject: [PATCH 14/48] BenchmarkScraperParse allocation speed fix --- pp/wal/benchmarks/scraper_benchmark.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pp/wal/benchmarks/scraper_benchmark.cpp b/pp/wal/benchmarks/scraper_benchmark.cpp index aa97fc6e37..ee87079baf 100644 --- a/pp/wal/benchmarks/scraper_benchmark.cpp +++ b/pp/wal/benchmarks/scraper_benchmark.cpp @@ -44,9 +44,12 @@ void BenchmarkScraperParse(benchmark::State& state) { std::ifstream t(get_file_name()); std::string str((std::istreambuf_iterator(t)), std::istreambuf_iterator()); + std::string tmp_str; + tmp_str.resize(str.size()); + for ([[maybe_unused]] auto _ : state) { + std::memcpy(tmp_str.data(), str.data(), str.size()); PrometheusScraper scraper; - auto tmp_str = str; std::ignore = scraper.parse(tmp_str, 0); } From b8c7aa493c327732c82b54a99dce7338b67be28b Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 3 Sep 2025 14:47:02 +0300 Subject: [PATCH 15/48] change codec 1234 to 0124 --- pp/wal/hashdex/scraper/scraper.h | 71 ++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index eda2366f18..9376355909 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -317,22 +317,18 @@ class Scraper { PROMPP_ALWAYS_INLINE static uint32_t read_val_partial(const char*& p, uint8_t sz) noexcept { - if (sz == 0) [[likely]] { - const uint32_t v = static_cast(p[0]); - p += 1; - return v; + if (sz == 0b01) [[likely]] { + return *p++; } - if (sz == 1) { - const uint32_t v = static_cast(static_cast(p[0])) | (static_cast(static_cast(p[1])) << 8); - p += 2; - return v; + if (sz == 0b00) { + return 0; } - if (sz == 2) { - const uint32_t v = static_cast(static_cast(p[0])) | (static_cast(static_cast(p[1])) << 8) | - (static_cast(static_cast(p[2])) << 16); - p += 3; + if (sz == 0b10) { + uint16_t v; + std::memcpy(&v, p, 2); + p += 2; return v; } @@ -480,6 +476,8 @@ class Scraper { } PROMPP_ALWAYS_INLINE void add_label(MarkedLabel label) noexcept { + static constexpr uint8_t szm[4] = {0, 1, 2, 4}; + const auto base_offset = this->buffer_.back().base_offset; if (label.name.is_reserved_name()) [[unlikely]] { @@ -490,27 +488,25 @@ class Scraper { } label.value.offset -= base_offset; - const uint8_t sz0 = encode_size(label.name.offset); - const uint8_t sz1 = encode_size(label.name.length); - const uint8_t sz2 = encode_size(label.value.offset); - const uint8_t sz3 = encode_size(label.value.length); - - const uint8_t layout = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); - - const uint32_t bytes_needed = sizeof(layout) + (sz0 + sz1 + sz2 + sz3) + 4; const uint32_t offset = bytes_count(); bytes_buffer_.resize(bytes_buffer_.size() + 17); char* out = bytes_buffer_.data() + offset; + char* layout_ptr = out++; + + const uint8_t sz0 = push_and_encode(out, label.name.offset); + out += szm[sz0]; + const uint8_t sz1 = push_and_encode(out, label.name.length); + out += szm[sz1]; + const uint8_t sz2 = push_and_encode(out, label.value.offset); + out += szm[sz2]; + const uint8_t sz3 = push_and_encode(out, label.value.length); + out += szm[sz3]; + + const uint8_t layout = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); - *out++ = static_cast(layout); + const uint32_t bytes_needed = out - layout_ptr; - std::memcpy(out, &label.name.offset, sz0 + 1); - out += sz0 + 1; - std::memcpy(out, &label.name.length, sz1 + 1); - out += sz1 + 1; - std::memcpy(out, &label.value.offset, sz2 + 1); - out += sz2 + 1; - std::memcpy(out, &label.value.length, sz3 + 1); + *layout_ptr = static_cast(layout); bytes_buffer_.resize(offset + bytes_needed); } @@ -572,9 +568,22 @@ class Scraper { return out + sizeof(T); } - PROMPP_ALWAYS_INLINE static uint8_t encode_size(uint32_t v) noexcept { - const uint32_t msb = (v == 0 ? 0 : 31 - std::countl_zero(v)); - return msb >> 3; + PROMPP_ALWAYS_INLINE static uint8_t push_and_encode(char* out, uint32_t v) noexcept { + if (v == 0) [[unlikely]] { + return 0b00; + } + if (v <= 0xFF) [[likely]] { + const uint8_t value = static_cast(v); + std::memcpy(out, &value, sizeof(value)); + return 0b01; + } + if (v <= 0xFFFF) [[unlikely]] { + const uint16_t value = static_cast(v); + std::memcpy(out, &value, sizeof(value)); + return 0b10; + } + std::memcpy(out, &v, sizeof(v)); + return 0b11; } }; From c6251c48781a593e15ca2e3303218ddcc9f90f82 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 3 Sep 2025 16:44:15 +0300 Subject: [PATCH 16/48] general opts --- pp/wal/hashdex/scraper/scraper.h | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 9376355909..9cc2558d48 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -2,6 +2,7 @@ #include +#include #include #include "bare_bones/algorithm.h" @@ -20,7 +21,7 @@ namespace PromPP::WAL::hashdex::scraper { template class Scraper { public: - [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { + [[nodiscard]] Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { labels_.reserve(255); default_timestamp_ = default_timestamp; @@ -416,10 +417,10 @@ class Scraper { using Iterator = typename Base::Iterator; using IteratorSentinel = typename Base::IteratorSentinel; - [[nodiscard]] Iterator begin(std::string_view buffer, Primitives::Timestamp default_ts) const noexcept { + [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer, Primitives::Timestamp default_ts) const noexcept { return {typename Base::Context{buffer, bytes_buffer_, default_ts}, this->buffer_.data(), this->items_count()}; } - [[nodiscard]] static IteratorSentinel end() noexcept { return {}; } + [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t bytes_count() const noexcept { return bytes_buffer_.size(); } @@ -436,7 +437,7 @@ class Scraper { this->buffer_.push_back(MarkedMetric{.base_offset = global_offset, .data_offset = bytes_count()}); } - PROMPP_ALWAYS_INLINE void add_count(uint32_t count) noexcept { + void add_count(uint32_t count) noexcept { constexpr uint8_t kContinueBit = 0x80; constexpr uint8_t kValueMask = 0x7F; @@ -475,7 +476,7 @@ class Scraper { } } - PROMPP_ALWAYS_INLINE void add_label(MarkedLabel label) noexcept { + void add_label(MarkedLabel label) noexcept { static constexpr uint8_t szm[4] = {0, 1, 2, 4}; const auto base_offset = this->buffer_.back().base_offset; @@ -511,7 +512,7 @@ class Scraper { bytes_buffer_.resize(offset + bytes_needed); } - PROMPP_ALWAYS_INLINE void add_sample(const MarkedSample& sample) noexcept { + void add_sample(const MarkedSample& sample) noexcept { const double val = sample.sample.value(); const bool has_ts = sample.has_ts; @@ -573,8 +574,7 @@ class Scraper { return 0b00; } if (v <= 0xFF) [[likely]] { - const uint8_t value = static_cast(v); - std::memcpy(out, &value, sizeof(value)); + *out = static_cast(v); return 0b01; } if (v <= 0xFFFF) [[unlikely]] { @@ -593,21 +593,19 @@ class Scraper { using Iterator = typename Base::Iterator; using IteratorSentinel = typename Base::IteratorSentinel; - [[nodiscard]] Iterator begin(std::string_view buffer) const noexcept { return {typename Base::Context{buffer}, this->buffer_.data(), this->items_count()}; } - [[nodiscard]] static IteratorSentinel end() noexcept { return {}; } + [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer) const noexcept { + return {typename Base::Context{buffer}, this->buffer_.data(), this->items_count()}; + } + [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } PROMPP_ALWAYS_INLINE void add(MarkedString metric_name, MarkedString text, Prometheus::MetadataType type) noexcept { this->buffer_.emplace_back(metric_name, text, type); } - void remove_item() noexcept { this->buffer_.pop_back(); } - void initialize(size_t reserve) noexcept { - this->buffer_.clear(); - this->buffer_.reserve(reserve); - } + PROMPP_ALWAYS_INLINE void remove_item() noexcept { this->buffer_.pop_back(); } }; - [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_metadata() { + [[nodiscard]] Error parse_metadata() { static constexpr auto get_metadata_type = [](Token token) PROMPP_LAMBDA_INLINE { if (token == Token::kHelp) { return Prometheus::MetadataType::kHelp; @@ -647,7 +645,7 @@ class Scraper { return Error::kNoError; } - [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_metric() { + [[nodiscard]] Error parse_metric() { labels_.clear(); bool have_metric_name = false; From ce8ac343018fbaba245a8433cf0c9a0edeef83f3 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 3 Sep 2025 17:49:26 +0300 Subject: [PATCH 17/48] add padding --- pp/wal/hashdex/scraper/scraper.h | 52 +++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 9cc2558d48..f926ef7722 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -32,6 +32,7 @@ class Scraper { switch (tokenizer.next()) { case Token::kEOF: case Token::kEOFWord: { + metric_buffer_.add_padding(); return parser_.validate_parse_result(); } @@ -193,22 +194,8 @@ class Scraper { // decode label for (uint32_t i = 0; i < labels_count; ++i) { - const uint8_t layout = static_cast(*ptr++); - const uint8_t sz0 = (layout >> 0) & 0x3; - const uint8_t sz1 = (layout >> 2) & 0x3; - const uint8_t sz2 = (layout >> 4) & 0x3; - const uint8_t sz3 = (layout >> 6) & 0x3; - - const uint32_t name_off = read_val_partial(ptr, sz0); - const uint32_t name_len = read_val_partial(ptr, sz1); - const uint32_t value_off = read_val_partial(ptr, sz2); - const uint32_t value_len = read_val_partial(ptr, sz3); - - if (name_len == 0 && name_off == 0) [[unlikely]] { - ts.label_set().append(Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); - } else { - ts.label_set().append(std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); - } + const uint32_t consumed_bytes = decode_label(ptr, base, ts.label_set()); + ptr += consumed_bytes; } // decode sample @@ -316,8 +303,7 @@ class Scraper { return v; } - PROMPP_ALWAYS_INLINE - static uint32_t read_val_partial(const char*& p, uint8_t sz) noexcept { + PROMPP_ALWAYS_INLINE static uint32_t read_val_partial(const char*& p, uint8_t sz) noexcept { if (sz == 0b01) [[likely]] { return *p++; } @@ -338,6 +324,31 @@ class Scraper { p += 4; return v; } + + template + PROMPP_ALWAYS_INLINE static uint32_t decode_label(const char* ptr, const char* base, LabelSet& labels) noexcept { + const char* start = ptr; + + const uint8_t layout = static_cast(*ptr++); + + const uint8_t sz0 = (layout >> 0) & 0x3; + const uint8_t sz1 = (layout >> 2) & 0x3; + const uint8_t sz2 = (layout >> 4) & 0x3; + const uint8_t sz3 = (layout >> 6) & 0x3; + + const uint32_t name_off = read_val_partial(ptr, sz0); + const uint32_t name_len = read_val_partial(ptr, sz1); + const uint32_t value_off = read_val_partial(ptr, sz2); + const uint32_t value_len = read_val_partial(ptr, sz3); + + if (name_len == 0 && name_off == 0) [[unlikely]] { + labels.append(Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); + } else { + labels.append(std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); + } + + return static_cast(ptr - start); + } }; class Metadata { @@ -559,6 +570,11 @@ class Scraper { bytes_buffer_.resize(offset + written); } + void add_padding() noexcept { + constexpr size_t kPaddingSizeBytes = 16; + bytes_buffer_.resize(bytes_buffer_.size() + kPaddingSizeBytes); + } + private: BareBones::Vector bytes_buffer_; From ef7f5414fbcb46df008a3fb08e80b48ef55a2dd2 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 3 Sep 2025 18:26:21 +0300 Subject: [PATCH 18/48] likelyhood optimizations --- pp/wal/hashdex/scraper/scraper.h | 176 ++++++++++++++++++++++--------- 1 file changed, 125 insertions(+), 51 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index f926ef7722..4ef91c5355 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -205,66 +205,57 @@ class Scraper { Primitives::Sample sample{}; sample.timestamp() = default_timestamp_; + double& val = sample.value(); + + if (type == 0b00000011) [[likely]] { // uint32_t + uint32_t x; + std::memcpy(&x, ptr, sizeof(x)); + ptr += sizeof(x); + val = static_cast(x); + } else if (type == 0b00001001) [[likely]] { // double + std::memcpy(&val, ptr, sizeof(val)); + ptr += sizeof(val); + } else [[unlikely]] { + switch (type) { + case 0b00000000: // zero + val = 0.0; + break; + + case 0b00000001: { // uint8 + val = static_cast(*ptr++); + break; + } - double val; - - switch (type) { - case 0b00000000: // zero - val = 0.0; - break; - - case 0b00000001: { // uint8 - val = static_cast(*ptr++); - break; - } - - case 0b00000010: { // uint16 - uint16_t x = static_cast(ptr[0]) | (static_cast(ptr[1]) << 8); - ptr += 2; - val = static_cast(x); - break; - } - - case 0b00000011: { // uint32 - uint32_t x; - std::memcpy(&x, ptr, sizeof(x)); - ptr += 4; - val = static_cast(x); - break; - } + case 0b00000010: { // uint16 + uint16_t x; + std::memcpy(&x, ptr, sizeof(x)); + ptr += sizeof(x); + val = static_cast(x); + break; + } - case 0b00000100: // NaN (normal) - val = Prometheus::kNormalNan; - break; + case 0b00000100: // NaN (normal) + val = Prometheus::kNormalNan; + break; - case 0b00001000: { // float32 - float x; - std::memcpy(&x, ptr, sizeof(x)); - ptr += sizeof(x); - val = static_cast(x); - break; - } + case 0b00001000: { // float32 + float x; + std::memcpy(&x, ptr, sizeof(x)); + ptr += sizeof(x); + val = static_cast(x); + break; + } - case 0b00001001: { // double - double x; - std::memcpy(&x, ptr, sizeof(x)); - ptr += sizeof(x); - val = x; - break; + default: + val = Prometheus::kStaleNan; + break; } - - default: - val = Prometheus::kStaleNan; - break; } if (has_ts) [[unlikely]] { - Primitives::Timestamp ts_val; - memcpy(&ts_val, ptr, sizeof(ts_val)); - sample.timestamp() = ts_val; + memcpy(&sample.timestamp(), ptr, sizeof(sample.timestamp())); } - sample.value() = val; ts.samples().emplace_back(sample); } @@ -329,7 +320,90 @@ class Scraper { PROMPP_ALWAYS_INLINE static uint32_t decode_label(const char* ptr, const char* base, LabelSet& labels) noexcept { const char* start = ptr; - const uint8_t layout = static_cast(*ptr++); + uint64_t chunk; + std::memcpy(&chunk, ptr, sizeof(chunk)); + uint8_t layout = static_cast(chunk & 0xFF); + chunk >>= 8; + ptr += 8; + + if (layout == 0b01010101) [[likely]] { + uint32_t packed = static_cast(chunk & 0xFFFFFFFFu); + uint8_t name_off = (packed >> 0) & 0xFF; + uint8_t name_len = (packed >> 8) & 0xFF; + uint8_t value_off = (packed >> 16) & 0xFF; + uint8_t value_len = (packed >> 24) & 0xFF; + + labels.append(std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); + + return 5; + } + + if ((layout & 0x0F) == 0) [[likely]] { + uint8_t sz2 = (layout >> 4) & 0x3; + uint8_t sz3 = (layout >> 6) & 0x3; + + uint32_t value_off = 0; + uint32_t value_len = 0; + size_t used = 0; + + // value_off + switch (sz2) { + case 0b00: { + value_off = 0; + break; + } + case 0b01: { + value_off = static_cast(chunk & 0xFF); + used += 1; + chunk >>= 8; + break; + } + case 0b10: { + value_off = static_cast(chunk & 0xFFFF); + used += 2; + chunk >>= 16; + break; + } + default: { + value_off = static_cast(chunk & 0xFFFFFFFFu); + used += 4; + chunk >>= 32; + break; + } + } + + // value_len + switch (sz3) { + case 0b00: { + value_len = 0; + break; + } + case 0b01: { + value_len = static_cast(chunk & 0xFF); + used += 1; + break; + } + case 0b10: { + value_len = static_cast(chunk & 0xFFFF); + used += 2; + break; + } + default: { + if (used + 4 <= 7) [[likely]] { + value_len = static_cast(chunk & 0xFFFFFFFFu); + used += 4; + } else [[unlikely]] { + std::memcpy(&value_len, start + 1 + used, 4); + used += 4; + } + break; + } + } + + labels.append(Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); + + return 1 + used; + } const uint8_t sz0 = (layout >> 0) & 0x3; const uint8_t sz1 = (layout >> 2) & 0x3; From 4a8dac4466ff08e5bb626fed142af7cb3b965c42 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Thu, 4 Sep 2025 16:42:26 +0300 Subject: [PATCH 19/48] parse_metric reorganize --- pp/wal/hashdex/scraper/scraper.h | 37 +++++++++++++++++--------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 4ef91c5355..4e6a5ef2c5 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -54,7 +54,6 @@ class Scraper { case Token::kMetricName: case Token::kBraceOpen: { if (const auto error = parse_metric(); error != Error::kNoError) [[unlikely]] { - metric_buffer_.remove_item(); return error; } @@ -511,11 +510,6 @@ class Scraper { [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return this->buffer_.allocated_memory() + bytes_buffer_.allocated_memory(); } - PROMPP_ALWAYS_INLINE void remove_item() noexcept { - bytes_buffer_.resize(this->buffer_.back().data_offset); - this->buffer_.pop_back(); - } - PROMPP_ALWAYS_INLINE void add_hash(uint64_t hash) noexcept { this->buffer_.back().hash = hash; } PROMPP_ALWAYS_INLINE void add_metric(uint32_t global_offset) noexcept { @@ -738,10 +732,13 @@ class Scraper { [[nodiscard]] Error parse_metric() { labels_.clear(); + marked_sample_ = {}; + marked_sample_.sample.timestamp() = default_timestamp_; + bool have_metric_name = false; auto& tokenizer = parser_.tokenizer(); - metric_buffer_.add_metric(tokenizer.token_str().data() - tokenizer.buffer().data()); + const uint32_t metric_offset = tokenizer.token_str().data() - tokenizer.buffer().data(); if (tokenizer.token() == Token::kMetricName) [[likely]] { labels_.push_back(MarkedLabel{.value = MarkedString::create(tokenizer.token_str(), tokenizer.buffer())}); @@ -766,9 +763,16 @@ class Scraper { return Error::kNoMetricName; } - process_labels_buffer(); + auto error = parse_metric_suffix(); + + if (error == Error::kNoError) [[likely]] { + metric_buffer_.add_metric(metric_offset); + + process_labels_buffer(); + metric_buffer_.add_sample(marked_sample_); + } - return parse_metric_suffix(); + return error; } [[nodiscard]] PROMPP_ALWAYS_INLINE Error tokenize_label_set(bool& have_metric_name) noexcept { @@ -856,28 +860,26 @@ class Scraper { return Error::kUnexpectedToken; } - MarkedSample sample{.sample = {default_timestamp_, {}}}; - if (const auto error = parse_sample(sample); error != Error::kNoError) { + if (const auto error = parse_sample(); error != Error::kNoError) { return error; } - metric_buffer_.add_sample(sample); return parser_.validate_parse_sample_result(); } - [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_sample(MarkedSample& sample) noexcept { + [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_sample() noexcept { auto& tokenizer = parser_.tokenizer(); - if (!parse_numeric_value(tokenizer.token_str(), sample.sample.value())) [[unlikely]] { + if (!parse_numeric_value(tokenizer.token_str(), marked_sample_.sample.value())) [[unlikely]] { return Error::kInvalidValue; } - if (std::isnan(sample.sample.value())) [[unlikely]] { - sample.sample.value() = Prometheus::kNormalNan; + if (std::isnan(marked_sample_.sample.value())) [[unlikely]] { + marked_sample_.sample.value() = Prometheus::kNormalNan; } tokenizer.next_non_whitespace(); - return parser_.parse_timestamp(sample.sample.timestamp(), sample.has_ts); + return parser_.parse_timestamp(marked_sample_.sample.timestamp(), marked_sample_.has_ts); } PROMPP_ALWAYS_INLINE void process_labels_buffer() noexcept { @@ -914,6 +916,7 @@ class Scraper { MetadataMarkupBuffer metadata_buffer_; BareBones::Vector labels_; Primitives::Timestamp default_timestamp_{}; + MarkedSample marked_sample_{}; }; using PrometheusScraper = Scraper; From 0f93e8f13c7165a39e84c71bcef55b73215cc859 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Thu, 4 Sep 2025 21:40:17 +0300 Subject: [PATCH 20/48] 1ms on read optimization --- pp/wal/hashdex/scraper/scraper.h | 120 +++++++++++++++++-------------- 1 file changed, 65 insertions(+), 55 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 4e6a5ef2c5..11ece6e081 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -506,7 +506,19 @@ class Scraper { } [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t bytes_count() const noexcept { return bytes_buffer_.size(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t bytes_count() const noexcept { return bytes_count_; } + void bytes_enlarge(uint32_t new_size) noexcept { + assert(new_size > bytes_count_); + if (new_size > bytes_buffer_.size()) [[likely]] { + bytes_buffer_.resize(new_size); + } + bytes_count_ = new_size; + } + + void bytes_shrink(uint32_t new_size) noexcept { + // assert(new_size <= bytes_count_); + bytes_count_ = new_size; + } [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return this->buffer_.allocated_memory() + bytes_buffer_.allocated_memory(); } @@ -520,46 +532,38 @@ class Scraper { constexpr uint8_t kContinueBit = 0x80; constexpr uint8_t kValueMask = 0x7F; - const uint32_t offset = bytes_count(); + uint64_t packed; + int bytes_written = 0; if (count < (1u << 7)) [[likely]] { - bytes_buffer_.resize(bytes_buffer_.size() + 1); - char* out = bytes_buffer_.data() + offset; - *out = static_cast(count); + packed = static_cast(count); + bytes_written = 1; } else if (count < (1u << 14)) { - bytes_buffer_.resize(bytes_buffer_.size() + 2); - char* out = bytes_buffer_.data() + offset; - *out++ = static_cast((count & kValueMask) | kContinueBit); - *out = static_cast(count >> 7); + packed = (static_cast((count & kValueMask) | kContinueBit)) | (static_cast(count >> 7) << 8); + bytes_written = 2; } else if (count < (1u << 21)) { - bytes_buffer_.resize(bytes_buffer_.size() + 3); - char* out = bytes_buffer_.data() + offset; - *out++ = static_cast((count & kValueMask) | kContinueBit); - *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); - *out = static_cast(count >> 14); + packed = (static_cast((count & kValueMask) | kContinueBit)) | (static_cast(((count >> 7) & kValueMask) | kContinueBit) << 8) | + (static_cast(count >> 14) << 16); + bytes_written = 3; } else if (count < (1u << 28)) { - bytes_buffer_.resize(bytes_buffer_.size() + 4); - char* out = bytes_buffer_.data() + offset; - *out++ = static_cast((count & kValueMask) | kContinueBit); - *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); - *out++ = static_cast(((count >> 14) & kValueMask) | kContinueBit); - *out = static_cast(count >> 21); + packed = (static_cast((count & kValueMask) | kContinueBit)) | (static_cast(((count >> 7) & kValueMask) | kContinueBit) << 8) | + (static_cast(((count >> 14) & kValueMask) | kContinueBit) << 16) | (static_cast(count >> 21) << 24); + bytes_written = 4; } else { - bytes_buffer_.resize(bytes_buffer_.size() + 5); - char* out = bytes_buffer_.data() + offset; - *out++ = static_cast((count & kValueMask) | kContinueBit); - *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); - *out++ = static_cast(((count >> 14) & kValueMask) | kContinueBit); - *out++ = static_cast(((count >> 21) & kValueMask) | kContinueBit); - *out = static_cast(count >> 28); + packed = (static_cast((count & kValueMask) | kContinueBit)) | (static_cast(((count >> 7) & kValueMask) | kContinueBit) << 8) | + (static_cast(((count >> 14) & kValueMask) | kContinueBit) << 16) | + (static_cast(((count >> 21) & kValueMask) | kContinueBit) << 24) | (static_cast(count >> 28) << 32); + bytes_written = 5; } + + const uint32_t offset = bytes_count(); + std::memcpy(bytes_buffer_.data() + offset, &packed, sizeof(packed)); + bytes_shrink(offset + bytes_written); } - void add_label(MarkedLabel label) noexcept { + void add_label(MarkedLabel label, uint32_t base_offset) noexcept { static constexpr uint8_t szm[4] = {0, 1, 2, 4}; - const auto base_offset = this->buffer_.back().base_offset; - if (label.name.is_reserved_name()) [[unlikely]] { label.name.offset = 0; label.name.length = 0; @@ -568,10 +572,10 @@ class Scraper { } label.value.offset -= base_offset; + // bytes_enlarge(offset + 17); const uint32_t offset = bytes_count(); - bytes_buffer_.resize(bytes_buffer_.size() + 17); char* out = bytes_buffer_.data() + offset; - char* layout_ptr = out++; + char* start = out++; const uint8_t sz0 = push_and_encode(out, label.name.offset); out += szm[sz0]; @@ -582,22 +586,17 @@ class Scraper { const uint8_t sz3 = push_and_encode(out, label.value.length); out += szm[sz3]; - const uint8_t layout = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); + *start = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); - const uint32_t bytes_needed = out - layout_ptr; - - *layout_ptr = static_cast(layout); - - bytes_buffer_.resize(offset + bytes_needed); + const uint32_t written = static_cast(out - start); + bytes_shrink(offset + written); } void add_sample(const MarkedSample& sample) noexcept { const double val = sample.sample.value(); const bool has_ts = sample.has_ts; - constexpr uint32_t max_sample_bytes = 1 + sizeof(sample.sample); const uint32_t offset = bytes_count(); - bytes_buffer_.resize(offset + max_sample_bytes); char* out = bytes_buffer_.data() + offset; char* start = out; @@ -635,16 +634,17 @@ class Scraper { } const uint32_t written = static_cast(out - start); - bytes_buffer_.resize(offset + written); + bytes_shrink(offset + written); } void add_padding() noexcept { constexpr size_t kPaddingSizeBytes = 16; - bytes_buffer_.resize(bytes_buffer_.size() + kPaddingSizeBytes); + bytes_enlarge(bytes_count() + kPaddingSizeBytes); } private: - BareBones::Vector bytes_buffer_; + BareBones::Vector bytes_buffer_{}; + uint32_t bytes_count_ = 0; template PROMPP_ALWAYS_INLINE static char* write_marker_and_value(char* out, uint8_t marker, bool has_ts, const T& val) noexcept { @@ -685,8 +685,6 @@ class Scraper { PROMPP_ALWAYS_INLINE void add(MarkedString metric_name, MarkedString text, Prometheus::MetadataType type) noexcept { this->buffer_.emplace_back(metric_name, text, type); } - - PROMPP_ALWAYS_INLINE void remove_item() noexcept { this->buffer_.pop_back(); } }; [[nodiscard]] Error parse_metadata() { @@ -768,14 +766,14 @@ class Scraper { if (error == Error::kNoError) [[likely]] { metric_buffer_.add_metric(metric_offset); - process_labels_buffer(); + process_labels_buffer(metric_offset); metric_buffer_.add_sample(marked_sample_); } return error; } - [[nodiscard]] PROMPP_ALWAYS_INLINE Error tokenize_label_set(bool& have_metric_name) noexcept { + [[nodiscard]] Error tokenize_label_set(bool& have_metric_name) noexcept { auto& tokenizer = parser_.tokenizer(); tokenizer.next_non_whitespace(); @@ -817,7 +815,7 @@ class Scraper { return tokenizer.token() == Token::kBraceClose ? Error::kNoError : Error::kUnexpectedToken; } - [[nodiscard]] PROMPP_ALWAYS_INLINE Error get_label_name(MarkedString& label_name) const noexcept { + [[nodiscard]] Error get_label_name(MarkedString& label_name) const noexcept { auto& tokenizer = parser_.tokenizer(); if (tokenizer.token() == Token::kLabelName) [[likely]] { @@ -831,7 +829,7 @@ class Scraper { return Error::kUnexpectedToken; } - [[nodiscard]] PROMPP_ALWAYS_INLINE Error get_quoted_value(MarkedString& string) const noexcept { + [[nodiscard]] Error get_quoted_value(MarkedString& string) const noexcept { auto& tokenizer = parser_.tokenizer(); auto value = tokenizer.token_str(); @@ -855,7 +853,7 @@ class Scraper { return Error::kNoError; } - [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_metric_suffix() noexcept { + [[nodiscard]] Error parse_metric_suffix() noexcept { if (!parser_.is_value_token()) [[unlikely]] { return Error::kUnexpectedToken; } @@ -867,7 +865,7 @@ class Scraper { return parser_.validate_parse_sample_result(); } - [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_sample() noexcept { + [[nodiscard]] Error parse_sample() noexcept { auto& tokenizer = parser_.tokenizer(); if (!parse_numeric_value(tokenizer.token_str(), marked_sample_.sample.value())) [[unlikely]] { @@ -882,18 +880,22 @@ class Scraper { return parser_.parse_timestamp(marked_sample_.sample.timestamp(), marked_sample_.has_ts); } - PROMPP_ALWAYS_INLINE void process_labels_buffer() noexcept { + void process_labels_buffer(uint32_t offset) noexcept { sort_and_filter_labels(); append_labels_hash(); + const uint32_t bytes_offset = metric_buffer_.bytes_count(); + metric_buffer_.bytes_enlarge(bytes_offset + calculate_metric_prealloc_size(labels_.size())); + metric_buffer_.bytes_shrink(bytes_offset); + metric_buffer_.add_count(labels_.size()); for (const auto& label : labels_) { - metric_buffer_.add_label(label); + metric_buffer_.add_label(label, offset); } } - PROMPP_ALWAYS_INLINE void sort_and_filter_labels() noexcept { + void sort_and_filter_labels() noexcept { const auto it = std::remove_if(labels_.begin(), labels_.end(), [](const MarkedLabel& label) { return label.value.is_empty(); }); labels_.erase(it, labels_.end()); @@ -902,7 +904,7 @@ class Scraper { }); } - PROMPP_ALWAYS_INLINE void append_labels_hash() noexcept { + void append_labels_hash() noexcept { const auto& tokenizer = parser_.tokenizer(); BareBones::XXHash hash; for (const auto& label : labels_) { @@ -911,6 +913,14 @@ class Scraper { metric_buffer_.add_hash(hash.hash()); } + static PROMPP_ALWAYS_INLINE uint32_t calculate_metric_prealloc_size(uint32_t labels_count) noexcept { + constexpr uint32_t kCountVarintPreallocBytes = 8; + constexpr uint32_t kLabelPreallocBytes = 17; + constexpr uint32_t kSamplePreallocSize = 17; + + return kCountVarintPreallocBytes + labels_count * kLabelPreallocBytes + kSamplePreallocSize; + } + Parser parser_; MetricMarkupBuffer metric_buffer_; MetadataMarkupBuffer metadata_buffer_; From 1d8fd63ab439890842cc02d1014edece3057b23b Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Thu, 4 Sep 2025 23:01:53 +0300 Subject: [PATCH 21/48] somehow its faster --- pp/wal/hashdex/scraper/scraper.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 11ece6e081..870b928dbe 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -899,8 +899,16 @@ class Scraper { const auto it = std::remove_if(labels_.begin(), labels_.end(), [](const MarkedLabel& label) { return label.value.is_empty(); }); labels_.erase(it, labels_.end()); - std::sort(labels_.begin(), labels_.end(), [buffer = parser_.tokenizer().buffer()](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { - return a.name.view(buffer) < b.name.view(buffer); + std::sort(labels_.begin(), labels_.end(), [buffer = parser_.tokenizer().buffer()](const MarkedLabel& a, const MarkedLabel& b) { + // return a.name.view(buffer) < b.name.view(buffer); + const auto av = a.name.view(buffer); + const auto bv = b.name.view(buffer); + + const auto len = std::min(av.size(), bv.size()); + + const int cmp = std::strncmp(av.data(), bv.data(), len); + + return cmp < 0 || (cmp == 0 && av.size() < bv.size()); }); } From adc6a9fd7c588aa9173c6f7fac5e9074c89b49e2 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Fri, 5 Sep 2025 11:30:30 +0300 Subject: [PATCH 22/48] read opts --- pp/wal/hashdex/scraper/scraper.h | 38 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 870b928dbe..631eb82344 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -321,16 +321,13 @@ class Scraper { uint64_t chunk; std::memcpy(&chunk, ptr, sizeof(chunk)); - uint8_t layout = static_cast(chunk & 0xFF); - chunk >>= 8; - ptr += 8; + uint8_t layout = static_cast(chunk); if (layout == 0b01010101) [[likely]] { - uint32_t packed = static_cast(chunk & 0xFFFFFFFFu); - uint8_t name_off = (packed >> 0) & 0xFF; - uint8_t name_len = (packed >> 8) & 0xFF; - uint8_t value_off = (packed >> 16) & 0xFF; - uint8_t value_len = (packed >> 24) & 0xFF; + const uint8_t name_off = static_cast(chunk >> 8); + const uint8_t name_len = static_cast(chunk >> 16); + const uint8_t value_off = static_cast(chunk >> 24); + const uint8_t value_len = static_cast(chunk >> 32); labels.append(std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); @@ -338,8 +335,10 @@ class Scraper { } if ((layout & 0x0F) == 0) [[likely]] { - uint8_t sz2 = (layout >> 4) & 0x3; - uint8_t sz3 = (layout >> 6) & 0x3; + chunk >>= 8; + + const uint8_t sz2 = (layout >> 4) & 0x3; + const uint8_t sz3 = (layout >> 6) & 0x3; uint32_t value_off = 0; uint32_t value_len = 0; @@ -352,19 +351,19 @@ class Scraper { break; } case 0b01: { - value_off = static_cast(chunk & 0xFF); + value_off = static_cast(chunk); used += 1; chunk >>= 8; break; } case 0b10: { - value_off = static_cast(chunk & 0xFFFF); + value_off = static_cast(chunk); used += 2; chunk >>= 16; break; } default: { - value_off = static_cast(chunk & 0xFFFFFFFFu); + value_off = static_cast(chunk); used += 4; chunk >>= 32; break; @@ -378,22 +377,21 @@ class Scraper { break; } case 0b01: { - value_len = static_cast(chunk & 0xFF); + value_len = static_cast(chunk); used += 1; break; } case 0b10: { - value_len = static_cast(chunk & 0xFFFF); + value_len = static_cast(chunk); used += 2; break; } default: { - if (used + 4 <= 7) [[likely]] { - value_len = static_cast(chunk & 0xFFFFFFFFu); - used += 4; + used += 4; + if (used <= 7) [[likely]] { + value_len = static_cast(chunk); } else [[unlikely]] { std::memcpy(&value_len, start + 1 + used, 4); - used += 4; } break; } @@ -404,6 +402,8 @@ class Scraper { return 1 + used; } + ptr += 1; + const uint8_t sz0 = (layout >> 0) & 0x3; const uint8_t sz1 = (layout >> 2) & 0x3; const uint8_t sz2 = (layout >> 4) & 0x3; From 8249e7f07365edb92533e195d4485c5df44146a0 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Fri, 5 Sep 2025 13:56:21 +0300 Subject: [PATCH 23/48] LabelSet reserve to resize --- pp/wal/hashdex/scraper/scraper.h | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 631eb82344..9cffb91c17 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -189,11 +189,12 @@ class Scraper { // decode label count uint32_t labels_count = decode_varint(ptr); - ts.label_set().reserve(labels_count); + ts.label_set().resize(labels_count); // decode label + auto label_iter = ts.label_set().begin(); for (uint32_t i = 0; i < labels_count; ++i) { - const uint32_t consumed_bytes = decode_label(ptr, base, ts.label_set()); + const uint32_t consumed_bytes = decode_label(ptr, base, label_iter++); ptr += consumed_bytes; } @@ -315,8 +316,8 @@ class Scraper { return v; } - template - PROMPP_ALWAYS_INLINE static uint32_t decode_label(const char* ptr, const char* base, LabelSet& labels) noexcept { + template + PROMPP_ALWAYS_INLINE static uint32_t decode_label(const char* ptr, const char* base, LabelSetIter iter) noexcept { const char* start = ptr; uint64_t chunk; @@ -329,7 +330,7 @@ class Scraper { const uint8_t value_off = static_cast(chunk >> 24); const uint8_t value_len = static_cast(chunk >> 32); - labels.append(std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); + *iter = {std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)}; return 5; } @@ -397,7 +398,7 @@ class Scraper { } } - labels.append(Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); + *iter = {Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)}; return 1 + used; } @@ -415,9 +416,9 @@ class Scraper { const uint32_t value_len = read_val_partial(ptr, sz3); if (name_len == 0 && name_off == 0) [[unlikely]] { - labels.append(Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); + *iter = {Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)}; } else { - labels.append(std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); + *iter = {std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)}; } return static_cast(ptr - start); From 8d1ec651ba15ec9232ef4b67a6451f55e7dadf1e Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Fri, 5 Sep 2025 14:18:11 +0300 Subject: [PATCH 24/48] LabelSet reserve to resize --- pp/primitives/label_set.h | 1 + 1 file changed, 1 insertion(+) diff --git a/pp/primitives/label_set.h b/pp/primitives/label_set.h index 28a8aa0b3c..c268fea8fa 100644 --- a/pp/primitives/label_set.h +++ b/pp/primitives/label_set.h @@ -91,6 +91,7 @@ class BasicLabelSet { [[nodiscard]] PROMPP_ALWAYS_INLINE auto size() const noexcept { return labels_.size(); } PROMPP_ALWAYS_INLINE void reserve(size_t size) noexcept { labels_.reserve(size); } + PROMPP_ALWAYS_INLINE void resize(size_t size) noexcept { labels_.resize(size); } using iterator = typename Container::iterator; using const_iterator = typename Container::const_iterator; From 7795763a270d9d4b626731cadecfb7299503a1a0 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Tue, 9 Sep 2025 15:57:52 +0300 Subject: [PATCH 25/48] scraper fixes --- pp/wal/hashdex/scraper/scraper.h | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 9cffb91c17..ac85de6c52 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -204,7 +204,6 @@ class Scraper { uint8_t type = marker & 0b01111111; Primitives::Sample sample{}; - sample.timestamp() = default_timestamp_; double& val = sample.value(); if (type == 0b00000011) [[likely]] { // uint32_t @@ -254,6 +253,8 @@ class Scraper { if (has_ts) [[unlikely]] { memcpy(&sample.timestamp(), ptr, sizeof(sample.timestamp())); + } else { + sample.timestamp() = default_timestamp_; } ts.samples().emplace_back(sample); @@ -330,7 +331,8 @@ class Scraper { const uint8_t value_off = static_cast(chunk >> 24); const uint8_t value_len = static_cast(chunk >> 32); - *iter = {std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)}; + //*iter = {std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)}; + std::construct_at(iter, std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); return 5; } @@ -398,7 +400,8 @@ class Scraper { } } - *iter = {Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)}; + //*iter = {Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)}; + std::construct_at(iter, Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); return 1 + used; } @@ -416,9 +419,9 @@ class Scraper { const uint32_t value_len = read_val_partial(ptr, sz3); if (name_len == 0 && name_off == 0) [[unlikely]] { - *iter = {Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)}; + std::construct_at(iter, Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); } else { - *iter = {std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)}; + std::construct_at(iter, std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); } return static_cast(ptr - start); @@ -573,7 +576,6 @@ class Scraper { } label.value.offset -= base_offset; - // bytes_enlarge(offset + 17); const uint32_t offset = bytes_count(); char* out = bytes_buffer_.data() + offset; char* start = out++; @@ -900,17 +902,8 @@ class Scraper { const auto it = std::remove_if(labels_.begin(), labels_.end(), [](const MarkedLabel& label) { return label.value.is_empty(); }); labels_.erase(it, labels_.end()); - std::sort(labels_.begin(), labels_.end(), [buffer = parser_.tokenizer().buffer()](const MarkedLabel& a, const MarkedLabel& b) { - // return a.name.view(buffer) < b.name.view(buffer); - const auto av = a.name.view(buffer); - const auto bv = b.name.view(buffer); - - const auto len = std::min(av.size(), bv.size()); - - const int cmp = std::strncmp(av.data(), bv.data(), len); - - return cmp < 0 || (cmp == 0 && av.size() < bv.size()); - }); + std::sort(labels_.begin(), labels_.end(), + [buffer = parser_.tokenizer().buffer()](const MarkedLabel& a, const MarkedLabel& b) { return a.name.view(buffer) < b.name.view(buffer); }); } void append_labels_hash() noexcept { From 95d451968318ad351c2b4555680558ff8a1d77cd Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 10 Sep 2025 10:10:26 +0300 Subject: [PATCH 26/48] sample encoding + tests --- pp/wal/hashdex/scraper/encoding.h | 137 ++++++++++ pp/wal/hashdex/scraper/encoding_tests.cpp | 235 ++++++++++++++++++ pp/wal/hashdex/scraper/scraper.h | 288 ++++++++-------------- 3 files changed, 478 insertions(+), 182 deletions(-) create mode 100644 pp/wal/hashdex/scraper/encoding.h create mode 100644 pp/wal/hashdex/scraper/encoding_tests.cpp diff --git a/pp/wal/hashdex/scraper/encoding.h b/pp/wal/hashdex/scraper/encoding.h new file mode 100644 index 0000000000..8d2d0fa540 --- /dev/null +++ b/pp/wal/hashdex/scraper/encoding.h @@ -0,0 +1,137 @@ +#pragma once + +#include +#include +#include + +#include "primitives/sample.h" +#include "prometheus/value.h" + +namespace PromPP::WAL::hashdex::scraper::encoding { +enum class SampleValueType : uint8_t { kUint32 = 0b0000'0000, kDouble, kUint8, kUint16, kFloat, kZero, kNaN }; + +[[nodiscard]] PROMPP_LAMBDA_INLINE inline SampleValueType value_type(const double val) noexcept { + if (std::isnan(val)) [[unlikely]] { + return SampleValueType::kNaN; + } + if (val == 0.0) [[unlikely]] { + return SampleValueType::kZero; + } + + if (std::trunc(val) == val && val > 0.0) [[likely]] { + const auto uval = static_cast(val); + if (uval <= std::numeric_limits::max()) { + return SampleValueType::kUint8; + } + if (uval <= std::numeric_limits::max()) { + return SampleValueType::kUint16; + } + if (uval <= std::numeric_limits::max()) { + return SampleValueType::kUint32; + } + } + + if (const auto f = static_cast(val); static_cast(f) == val) [[unlikely]] { + return SampleValueType::kFloat; + } + + return SampleValueType::kDouble; +} + +struct LayoutMarker { + uint8_t raw; + + [[nodiscard]] PROMPP_LAMBDA_INLINE bool has_timestamp() const noexcept { return raw & 0b1000'0000; } + + [[nodiscard]] PROMPP_LAMBDA_INLINE uint8_t count_size_in_bytes() const noexcept { return (raw >> 4) & 0b0000'0111; } + + [[nodiscard]] PROMPP_LAMBDA_INLINE SampleValueType value_type() const noexcept { return static_cast(raw & 0b0000'1111); } + + static PROMPP_LAMBDA_INLINE LayoutMarker make(bool has_ts, uint32_t labels_count, SampleValueType value_type) noexcept { + const uint8_t bytes_for_count = (std::bit_width(labels_count) + 7) / 8; + + return {static_cast((has_ts ? 0b1000'0000 : 0) | (bytes_for_count << 4) | (static_cast(value_type) & 0b0000'1111))}; + } +}; + +class SampleCodec { + public: + static PROMPP_LAMBDA_INLINE char* encode(char* out, const LayoutMarker layout, Primitives::Sample sample) { + using encoding::SampleValueType; + + const double val = sample.value(); + if (const auto type = layout.value_type(); type == SampleValueType::kUint32) [[likely]] { + out = write_value(out, static_cast(val)); + } else if (type == SampleValueType::kDouble) [[likely]] { + out = write_value(out, val); + } else if (type == SampleValueType::kUint8) [[unlikely]] { + out = write_value(out, static_cast(val)); + } else if (type == SampleValueType::kUint16) [[unlikely]] { + out = write_value(out, static_cast(val)); + } else if (type == SampleValueType::kFloat) [[unlikely]] { + out = write_value(out, static_cast(val)); + } + + if (layout.has_timestamp()) [[unlikely]] { + out = write_value(out, sample.timestamp()); + } + + return out; + } + + struct DecodeResult { + const char* next; + Primitives::Sample sample; + }; + + static PROMPP_LAMBDA_INLINE DecodeResult decode(const char* in, const LayoutMarker layout, int64_t default_ts) { + using encoding::SampleValueType; + + DecodeResult result{}; + + double& val = result.sample.value(); + uint64_t chunk; + std::memcpy(&chunk, in, sizeof(chunk)); + + if (const auto type = layout.value_type(); type == SampleValueType::kUint32) [[likely]] { + val = static_cast(static_cast(chunk)); + in += sizeof(uint32_t); + } else if (type == SampleValueType::kDouble) [[likely]] { + val = std::bit_cast(chunk); + in += sizeof(double); + } else if (type == SampleValueType::kUint8) [[unlikely]] { + val = static_cast(static_cast(chunk)); + in += sizeof(uint8_t); + } else if (type == SampleValueType::kUint16) [[unlikely]] { + val = static_cast(static_cast(chunk)); + in += sizeof(uint16_t); + } else if (type == SampleValueType::kFloat) [[unlikely]] { + val = static_cast(std::bit_cast(static_cast(chunk))); + in += sizeof(float); + } else if (type == SampleValueType::kZero) [[unlikely]] { + val = 0.0; + } else { + val = Prometheus::kNormalNan; + } + + if (auto& timestamp = result.sample.timestamp(); layout.has_timestamp()) [[unlikely]] { + std::memcpy(×tamp, in, sizeof(timestamp)); + in += sizeof(timestamp); + } else { + timestamp = default_ts; + } + + result.next = in; + + return result; + } + + private: + template + PROMPP_ALWAYS_INLINE static char* write_value(char* out, const T& val) noexcept { + std::memcpy(out, &val, sizeof(T)); + return out + sizeof(T); + } +}; + +} // namespace PromPP::WAL::hashdex::scraper::encoding \ No newline at end of file diff --git a/pp/wal/hashdex/scraper/encoding_tests.cpp b/pp/wal/hashdex/scraper/encoding_tests.cpp new file mode 100644 index 0000000000..830361f109 --- /dev/null +++ b/pp/wal/hashdex/scraper/encoding_tests.cpp @@ -0,0 +1,235 @@ +#include + +#include +#include +#include +#include + +#include "encoding.h" +#include "primitives/sample.h" +#include "prometheus/value.h" + +namespace { + +using namespace PromPP::WAL::hashdex::scraper::encoding; +using PromPP::Primitives::Sample; +using PromPP::Primitives::Timestamp; + +struct ValueTypeCase { + double input; + SampleValueType expected; +}; + +class ValueTypeFixture : public testing::TestWithParam {}; + +TEST_P(ValueTypeFixture, ClassifyValueCorrectly) { + // Arrange + + // Act + const auto actual = value_type(GetParam().input); + + // Assert + EXPECT_EQ(actual, GetParam().expected); +} + +INSTANTIATE_TEST_SUITE_P(ValueTypeTests, + ValueTypeFixture, + testing::Values(ValueTypeCase{.input = PromPP::Prometheus::kNormalNan, .expected = SampleValueType::kNaN}, + ValueTypeCase{.input = PromPP::Prometheus::kStaleNan, .expected = SampleValueType::kNaN}, + ValueTypeCase{.input = std::numeric_limits::quiet_NaN(), .expected = SampleValueType::kNaN}, + + ValueTypeCase{.input = 0.0, .expected = SampleValueType::kZero}, + ValueTypeCase{.input = -0.0, .expected = SampleValueType::kZero}, + + ValueTypeCase{.input = 1.0, .expected = SampleValueType::kUint8}, + ValueTypeCase{.input = 255.0, .expected = SampleValueType::kUint8}, + + ValueTypeCase{.input = 256.0, .expected = SampleValueType::kUint16}, + ValueTypeCase{.input = 65535.0, .expected = SampleValueType::kUint16}, + + ValueTypeCase{.input = 65536.0, .expected = SampleValueType::kUint32}, + ValueTypeCase{.input = 4294967295.0, .expected = SampleValueType::kUint32}, + + ValueTypeCase{.input = 3.5, .expected = SampleValueType::kFloat}, + ValueTypeCase{.input = -42.0, .expected = SampleValueType::kFloat}, + + ValueTypeCase{.input = 1e300, .expected = SampleValueType::kDouble}, + ValueTypeCase{.input = 3.141592653589793, .expected = SampleValueType::kDouble}, + ValueTypeCase{.input = 0.1, .expected = SampleValueType::kDouble})); + +class SampleCodecFixture : public testing::Test { + protected: + static constexpr size_t kBufSize = 64; + std::pair&> encode_into_buffer(const LayoutMarker layout, Sample sample) { + buf_.fill(0); + char* start = buf_.data(); + char* end = SampleCodec::encode(start, layout, sample); + return {end, buf_}; + } + + SampleCodec::DecodeResult encode_and_decode(const LayoutMarker layout, Sample sample, int64_t default_ts = -1) { + buf_.fill(0); + char* start = buf_.data(); + char* end = SampleCodec::encode(start, layout, sample); + + const auto res = SampleCodec::decode(start, layout, default_ts); + + EXPECT_EQ(res.next, end); + return res; + } + + std::array buf_{}; + const Timestamp default_ts_ = -1; +}; + +TEST_F(SampleCodecFixture, EncodeDecode_Uint32_WithTimestamp) { + // Arrange + Sample s{}; + s.value() = 123.0; + s.timestamp() = int64_t{42}; + + const auto layout = LayoutMarker::make(true, 0, SampleValueType::kUint32); + + // Act + auto res = encode_and_decode(layout, s, default_ts_); + + // Assert + EXPECT_DOUBLE_EQ(res.sample.value(), 123.0); + EXPECT_EQ(res.sample.timestamp(), 42); +} + +TEST_F(SampleCodecFixture, EncodeDecode_Double_WithoutTimestamp) { + // Arrange + Sample s{}; + s.value() = 3.141592653589793; + s.timestamp() = int64_t{12345}; + + const auto layout = LayoutMarker::make(false, 0, SampleValueType::kDouble); + + // Act + const auto res = encode_and_decode(layout, s, default_ts_); + + // Assert + EXPECT_DOUBLE_EQ(res.sample.value(), 3.141592653589793); + EXPECT_EQ(res.sample.timestamp(), default_ts_); +} + +TEST_F(SampleCodecFixture, EncodeDecode_Float_WithTimestamp) { + // Arrange + Sample s{}; + s.value() = 1.2345; + s.timestamp() = int64_t{111}; + + const auto layout = LayoutMarker::make(true, 0, SampleValueType::kFloat); + + // Act + const auto res = encode_and_decode(layout, s, default_ts_); + + // Assert + EXPECT_DOUBLE_EQ(res.sample.value(), static_cast(static_cast(1.2345))); + EXPECT_EQ(res.sample.timestamp(), 111); +} + +TEST_F(SampleCodecFixture, EncodeDecode_Uint8) { + // Arrange + Sample s{}; + s.value() = 255.0; + s.timestamp() = 7; + const auto layout = LayoutMarker::make(true, 0, SampleValueType::kUint8); + + // Act + const auto res = encode_and_decode(layout, s, default_ts_); + + // Assert + EXPECT_DOUBLE_EQ(res.sample.value(), 255.0); + EXPECT_EQ(res.sample.timestamp(), 7); +} + +TEST_F(SampleCodecFixture, EncodeDecode_Uint16) { + // Arrange + Sample s{}; + s.value() = 65535.0; + s.timestamp() = 9; + const auto layout = LayoutMarker::make(true, 0, SampleValueType::kUint16); + + // Act + const auto res = encode_and_decode(layout, s, default_ts_); + + // Assert + EXPECT_DOUBLE_EQ(res.sample.value(), 65535.0); + EXPECT_EQ(res.sample.timestamp(), 9); +} + +TEST_F(SampleCodecFixture, EncodeDecode_ZeroType_IgnoresOriginalValue) { + // Arrange + Sample s{}; + s.value() = 12345.0; + s.timestamp() = 222; + const auto layout = LayoutMarker::make(true, 0, SampleValueType::kZero); + + // Act + const auto res = encode_and_decode(layout, s, default_ts_); + + // Assert + EXPECT_DOUBLE_EQ(res.sample.value(), 0.0); + EXPECT_EQ(res.sample.timestamp(), 222); +} + +TEST_F(SampleCodecFixture, Decode_NaNType_ProducesNaN) { + // Arrange + Sample s{}; + s.value() = 777.0; + s.timestamp() = 333; + const auto layout = LayoutMarker::make(true, 0, SampleValueType::kNaN); + + // Act + const auto res = encode_and_decode(layout, s, default_ts_); + + // Assert + EXPECT_TRUE(std::isnan(res.sample.value())); + EXPECT_EQ(res.sample.timestamp(), 333); +} + +TEST_F(SampleCodecFixture, Decode_ReturnedPointerAdvancesCorrectly_NoTimestamp) { + // Arrange + Sample s{}; + s.value() = 42.0; + s.timestamp() = 1000; + + const auto layout = LayoutMarker::make(false, 0, SampleValueType::kUint32); + + buf_.fill(0); + char* start = buf_.data(); + char* end = SampleCodec::encode(start, layout, s); + + // Act + const auto res = SampleCodec::decode(start, layout, default_ts_); + + // Assert + EXPECT_EQ(res.next, end); + EXPECT_DOUBLE_EQ(res.sample.value(), 42.0); + EXPECT_EQ(res.sample.timestamp(), default_ts_); +} + +TEST_F(SampleCodecFixture, Decode_ReturnedPointerAdvancesCorrectly_WithTimestamp) { + // Arrange + Sample s{}; + s.value() = 4242.0; + s.timestamp() = 2048; + + const auto layout = LayoutMarker::make(true, 0, SampleValueType::kDouble); + + buf_.fill(0); + char* start = buf_.data(); + char* end = SampleCodec::encode(start, layout, s); + + // Act + const auto res = SampleCodec::decode(start, layout, default_ts_); + + // Assert + EXPECT_EQ(res.next, end); + EXPECT_DOUBLE_EQ(res.sample.value(), 4242.0); + EXPECT_EQ(res.sample.timestamp(), 2048); +} + +} // namespace \ No newline at end of file diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index ac85de6c52..737a0053fc 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -2,12 +2,13 @@ #include -#include #include #include "bare_bones/algorithm.h" +#include "bare_bones/bit.h" #include "bare_bones/vector.h" #include "bare_bones/xxhash.h" +#include "encoding.h" #include "parser.h" #include "prometheus/hashdex.h" #include "prometheus/metric.h" @@ -22,7 +23,10 @@ template class Scraper { public: [[nodiscard]] Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { + metric_buffer_.initialize(buffer.size() / 4); + metadata_buffer_.initialize(buffer.size() / 128); labels_.reserve(255); + default_timestamp_ = default_timestamp; auto& tokenizer = parser_.tokenizer(); @@ -118,7 +122,7 @@ class Scraper { uint32_t offset{std::numeric_limits::max()}; uint32_t length{std::numeric_limits::max()}; - [[nodiscard]] PROMPP_ALWAYS_INLINE static MarkedString create(const std::string_view& value, const std::string_view& buffer) noexcept { + [[nodiscard]] PROMPP_ALWAYS_INLINE static MarkedString create(std::string_view value, std::string_view buffer) noexcept { return { .offset = static_cast(value.data() - buffer.data()), .length = static_cast(value.size()), @@ -151,9 +155,9 @@ class Scraper { }; struct MarkedMetric { - uint64_t hash{}; - uint32_t base_offset{}; - uint32_t data_offset{}; + uint64_t hash; + uint32_t base_offset; + uint32_t data_offset; }; struct MarkedMetadata { @@ -171,7 +175,7 @@ class Scraper { struct Context { std::string_view buffer; const BareBones::Vector& bytes_buffer; - Primitives::Timestamp default_timestamp; + Primitives::Timestamp default_timestamp{}; }; Metric(const Context& ctx, const MarkedMetric* item) @@ -187,75 +191,19 @@ class Scraper { const char* ptr = reinterpret_cast(bytes_buffer_.data() + item_->data_offset); const char* base = buffer_.data() + item_->base_offset; - // decode label count - uint32_t labels_count = decode_varint(ptr); + uint32_t labels_count{}; + encoding::LayoutMarker layout{}; + + ptr = decode_varint(ptr, layout, labels_count); ts.label_set().resize(labels_count); - // decode label auto label_iter = ts.label_set().begin(); for (uint32_t i = 0; i < labels_count; ++i) { const uint32_t consumed_bytes = decode_label(ptr, base, label_iter++); ptr += consumed_bytes; } - // decode sample - uint8_t marker = static_cast(*ptr++); - bool has_ts = (marker & 0b10000000) != 0; - uint8_t type = marker & 0b01111111; - - Primitives::Sample sample{}; - double& val = sample.value(); - - if (type == 0b00000011) [[likely]] { // uint32_t - uint32_t x; - std::memcpy(&x, ptr, sizeof(x)); - ptr += sizeof(x); - val = static_cast(x); - } else if (type == 0b00001001) [[likely]] { // double - std::memcpy(&val, ptr, sizeof(val)); - ptr += sizeof(val); - } else [[unlikely]] { - switch (type) { - case 0b00000000: // zero - val = 0.0; - break; - - case 0b00000001: { // uint8 - val = static_cast(*ptr++); - break; - } - - case 0b00000010: { // uint16 - uint16_t x; - std::memcpy(&x, ptr, sizeof(x)); - ptr += sizeof(x); - val = static_cast(x); - break; - } - - case 0b00000100: // NaN (normal) - val = Prometheus::kNormalNan; - break; - - case 0b00001000: { // float32 - float x; - std::memcpy(&x, ptr, sizeof(x)); - ptr += sizeof(x); - val = static_cast(x); - break; - } - - default: - val = Prometheus::kStaleNan; - break; - } - } - - if (has_ts) [[unlikely]] { - memcpy(&sample.timestamp(), ptr, sizeof(sample.timestamp())); - } else { - sample.timestamp() = default_timestamp_; - } + auto [p, sample] = encoding::SampleCodec::decode(ptr, layout, default_timestamp_); ts.samples().emplace_back(sample); } @@ -266,33 +214,18 @@ class Scraper { const MarkedMetric* item_{}; Primitives::Timestamp default_timestamp_; - PROMPP_ALWAYS_INLINE static uint32_t decode_varint(const char*& ptr) noexcept { - uint32_t b0 = static_cast(*ptr++); - if ((b0 & 0x80) == 0) [[likely]] { - return b0; - } + PROMPP_ALWAYS_INLINE static const char* decode_varint(const char* ptr, encoding::LayoutMarker& layout, uint32_t& labels_count) noexcept { + uint64_t chunk; + std::memcpy(&chunk, ptr, sizeof(chunk)); - uint32_t b1 = static_cast(*ptr++); - uint32_t v = (b0 & 0x7F) | ((b1 & 0x7F) << 7); - if ((b1 & 0x80) == 0) { - return v; - } + layout = encoding::LayoutMarker{.raw = static_cast(chunk)}; - uint32_t b2 = static_cast(*ptr++); - v |= (b2 & 0x7F) << 14; - if ((b2 & 0x80) == 0) { - return v; - } + chunk >>= 8; + const uint64_t mask = (1ULL << BareBones::Bit::to_bits(layout.count_size_in_bytes())) - 1; - uint32_t b3 = static_cast(*ptr++); - v |= (b3 & 0x7F) << 21; - if ((b3 & 0x80) == 0) { - return v; - } + labels_count = static_cast(chunk & mask); - uint32_t b4 = static_cast(*ptr++); - v |= (b4 & 0x0F) << 28; - return v; + return ptr + 1 + layout.count_size_in_bytes(); } PROMPP_ALWAYS_INLINE static uint32_t read_val_partial(const char*& p, uint8_t sz) noexcept { @@ -331,7 +264,6 @@ class Scraper { const uint8_t value_off = static_cast(chunk >> 24); const uint8_t value_len = static_cast(chunk >> 32); - //*iter = {std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)}; std::construct_at(iter, std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); return 5; @@ -400,7 +332,6 @@ class Scraper { } } - //*iter = {Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)}; std::construct_at(iter, Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); return 1 + used; @@ -519,49 +450,37 @@ class Scraper { bytes_count_ = new_size; } - void bytes_shrink(uint32_t new_size) noexcept { - // assert(new_size <= bytes_count_); - bytes_count_ = new_size; - } + void bytes_shrink(uint32_t new_size) noexcept { bytes_count_ = new_size; } [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return this->buffer_.allocated_memory() + bytes_buffer_.allocated_memory(); } + PROMPP_ALWAYS_INLINE void initialize(size_t reserve_bytes) noexcept { + this->buffer_.clear(); + + const size_t bytes_buffer_reserve = (reserve_bytes / 3) * 2; + const size_t items_buffer_reserve = (bytes_buffer_reserve / 3) / sizeof(MarkedMetric); + + this->buffer_.reserve(items_buffer_reserve); + bytes_buffer_.reserve(bytes_buffer_reserve); + bytes_count_ = 0; + } + PROMPP_ALWAYS_INLINE void add_hash(uint64_t hash) noexcept { this->buffer_.back().hash = hash; } PROMPP_ALWAYS_INLINE void add_metric(uint32_t global_offset) noexcept { - this->buffer_.push_back(MarkedMetric{.base_offset = global_offset, .data_offset = bytes_count()}); + this->buffer_.push_back(MarkedMetric{.hash = {}, .base_offset = global_offset, .data_offset = bytes_count()}); } - void add_count(uint32_t count) noexcept { - constexpr uint8_t kContinueBit = 0x80; - constexpr uint8_t kValueMask = 0x7F; - - uint64_t packed; - int bytes_written = 0; - - if (count < (1u << 7)) [[likely]] { - packed = static_cast(count); - bytes_written = 1; - } else if (count < (1u << 14)) { - packed = (static_cast((count & kValueMask) | kContinueBit)) | (static_cast(count >> 7) << 8); - bytes_written = 2; - } else if (count < (1u << 21)) { - packed = (static_cast((count & kValueMask) | kContinueBit)) | (static_cast(((count >> 7) & kValueMask) | kContinueBit) << 8) | - (static_cast(count >> 14) << 16); - bytes_written = 3; - } else if (count < (1u << 28)) { - packed = (static_cast((count & kValueMask) | kContinueBit)) | (static_cast(((count >> 7) & kValueMask) | kContinueBit) << 8) | - (static_cast(((count >> 14) & kValueMask) | kContinueBit) << 16) | (static_cast(count >> 21) << 24); - bytes_written = 4; - } else { - packed = (static_cast((count & kValueMask) | kContinueBit)) | (static_cast(((count >> 7) & kValueMask) | kContinueBit) << 8) | - (static_cast(((count >> 14) & kValueMask) | kContinueBit) << 16) | - (static_cast(((count >> 21) & kValueMask) | kContinueBit) << 24) | (static_cast(count >> 28) << 32); - bytes_written = 5; - } + void add_layout(uint8_t layout) noexcept { + const uint32_t offset = bytes_count(); + *(bytes_buffer_.data() + offset) = layout; + bytes_shrink(offset + 1); + } + void add_count(uint32_t count) noexcept { + const uint32_t bytes_written = (std::bit_width(count) + 7) / 8; const uint32_t offset = bytes_count(); - std::memcpy(bytes_buffer_.data() + offset, &packed, sizeof(packed)); + std::memcpy(bytes_buffer_.data() + offset, &count, sizeof(count)); bytes_shrink(offset + bytes_written); } @@ -591,53 +510,17 @@ class Scraper { *start = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); - const uint32_t written = static_cast(out - start); + const auto written = static_cast(out - start); bytes_shrink(offset + written); } - void add_sample(const MarkedSample& sample) noexcept { - const double val = sample.sample.value(); - const bool has_ts = sample.has_ts; + void add_sample(encoding::LayoutMarker layout, const Primitives::Sample& sample) noexcept { + using encoding::SampleValueType; - const uint32_t offset = bytes_count(); - char* out = bytes_buffer_.data() + offset; - char* start = out; - - if (std::isnan(val)) [[unlikely]] { - *out++ = static_cast((has_ts ? 0b10000000 : 0) | 0b00000100); // NaN - } else if (val == 0.0) [[unlikely]] { - *out++ = static_cast((has_ts ? 0b10000000 : 0) | 0b00000000); // Zero - } else if (std::trunc(val) == val && val > 0.0) [[likely]] { - const uint64_t uval = static_cast(val); - if (uval <= std::numeric_limits::max()) { - const auto v = static_cast(uval); - out = write_marker_and_value(out, 0b00000001, has_ts, v); - } else if (uval <= std::numeric_limits::max()) { - const auto v = static_cast(uval); - out = write_marker_and_value(out, 0b00000010, has_ts, v); - } else if (uval <= std::numeric_limits::max()) { - const auto v = static_cast(uval); - out = write_marker_and_value(out, 0b00000011, has_ts, v); - } else { - out = write_marker_and_value(out, 0b00001001, has_ts, val); // double - } - } else { - float f = static_cast(val); - if (static_cast(f) == val) [[unlikely]] { - out = write_marker_and_value(out, 0b00001000, has_ts, f); // float - } else { - out = write_marker_and_value(out, 0b00001001, has_ts, val); // double - } - } - - if (has_ts) { - const auto ts = sample.sample.timestamp(); - std::memcpy(out, &ts, sizeof(ts)); - out += sizeof(ts); - } + char* data_ptr = bytes_buffer_.data(); + const char* end = encoding::SampleCodec::encode(data_ptr + bytes_count(), layout, sample); - const uint32_t written = static_cast(out - start); - bytes_shrink(offset + written); + bytes_shrink(end - data_ptr); } void add_padding() noexcept { @@ -649,13 +532,6 @@ class Scraper { BareBones::Vector bytes_buffer_{}; uint32_t bytes_count_ = 0; - template - PROMPP_ALWAYS_INLINE static char* write_marker_and_value(char* out, uint8_t marker, bool has_ts, const T& val) noexcept { - *out++ = static_cast((has_ts ? 0b10000000 : 0) | marker); - std::memcpy(out, &val, sizeof(T)); - return out + sizeof(T); - } - PROMPP_ALWAYS_INLINE static uint8_t push_and_encode(char* out, uint32_t v) noexcept { if (v == 0) [[unlikely]] { return 0b00; @@ -665,7 +541,7 @@ class Scraper { return 0b01; } if (v <= 0xFFFF) [[unlikely]] { - const uint16_t value = static_cast(v); + const auto value = static_cast(v); std::memcpy(out, &value, sizeof(value)); return 0b10; } @@ -685,6 +561,12 @@ class Scraper { } [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } + PROMPP_ALWAYS_INLINE void initialize(size_t reserve_bytes) noexcept { + this->buffer_.clear(); + const size_t items_buffer_reserve = reserve_bytes / sizeof(MarkedMetric); + this->buffer_.reserve(items_buffer_reserve); + } + PROMPP_ALWAYS_INLINE void add(MarkedString metric_name, MarkedString text, Prometheus::MetadataType type) noexcept { this->buffer_.emplace_back(metric_name, text, type); } @@ -767,10 +649,7 @@ class Scraper { auto error = parse_metric_suffix(); if (error == Error::kNoError) [[likely]] { - metric_buffer_.add_metric(metric_offset); - - process_labels_buffer(metric_offset); - metric_buffer_.add_sample(marked_sample_); + encode_metric_data(metric_offset); } return error; @@ -883,14 +762,59 @@ class Scraper { return parser_.parse_timestamp(marked_sample_.sample.timestamp(), marked_sample_.has_ts); } - void process_labels_buffer(uint32_t offset) noexcept { - sort_and_filter_labels(); - append_labels_hash(); + void encode_metric_data(uint32_t metric_offset) noexcept { + metric_buffer_.add_metric(metric_offset); const uint32_t bytes_offset = metric_buffer_.bytes_count(); metric_buffer_.bytes_enlarge(bytes_offset + calculate_metric_prealloc_size(labels_.size())); metric_buffer_.bytes_shrink(bytes_offset); + const encoding::LayoutMarker layout = create_layout_marker(); + metric_buffer_.add_layout(layout.raw); + + process_labels_buffer(metric_offset); + metric_buffer_.add_sample(layout, marked_sample_.sample); + } + + [[nodiscard]] encoding::LayoutMarker create_layout_marker() noexcept { + using encoding::LayoutMarker; + using encoding::SampleValueType; + + const uint32_t labels_count = labels_.size(); + const double val = marked_sample_.sample.value(); + + if (std::isnan(val)) [[unlikely]] { + return LayoutMarker::make(marked_sample_.has_ts, labels_count, SampleValueType::kNaN); + } + if (val == 0.0) [[unlikely]] { + return LayoutMarker::make(marked_sample_.has_ts, labels_count, SampleValueType::kZero); + } + + if (std::trunc(val) == val && val > 0.0) [[likely]] { + const auto uval = static_cast(val); + if (uval <= std::numeric_limits::max()) { + return LayoutMarker::make(marked_sample_.has_ts, labels_count, SampleValueType::kUint8); + } + if (uval <= std::numeric_limits::max()) { + return LayoutMarker::make(marked_sample_.has_ts, labels_count, SampleValueType::kUint16); + } + if (uval <= std::numeric_limits::max()) { + return LayoutMarker::make(marked_sample_.has_ts, labels_count, SampleValueType::kUint32); + } + } + + const auto f = static_cast(val); + if (static_cast(f) == val) [[unlikely]] { + return LayoutMarker::make(marked_sample_.has_ts, labels_count, SampleValueType::kFloat); + } + + return LayoutMarker::make(marked_sample_.has_ts, labels_count, SampleValueType::kDouble); + } + + void process_labels_buffer(uint32_t offset) noexcept { + sort_and_filter_labels(); + append_labels_hash(); + metric_buffer_.add_count(labels_.size()); for (const auto& label : labels_) { From c4acfaf74e03958d34a3df99598e9d025e65a01f Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 10 Sep 2025 16:51:43 +0300 Subject: [PATCH 27/48] scraper fixes --- pp/wal/hashdex/scraper/scraper.h | 97 +++++++++++--------------------- 1 file changed, 32 insertions(+), 65 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 737a0053fc..5a22d5cfa4 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -79,11 +79,11 @@ class Scraper { public: explicit MetricsWrapper(const Scraper& scraper) : scraper_(scraper) {} - [[nodiscard]] PROMPP_LAMBDA_INLINE uint32_t size() const noexcept { return scraper_.metric_buffer_.items_count(); } - [[nodiscard]] PROMPP_LAMBDA_INLINE auto begin() const noexcept { + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t size() const noexcept { return scraper_.metric_buffer_.items_count(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE auto begin() const noexcept { return scraper_.metric_buffer_.begin(scraper_.parser_.tokenizer().buffer(), scraper_.default_timestamp()); } - [[nodiscard]] PROMPP_LAMBDA_INLINE static auto end() noexcept { return MetricMarkupBuffer::end(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE static auto end() noexcept { return MetricMarkupBuffer::end(); } private: const Scraper& scraper_; @@ -93,17 +93,17 @@ class Scraper { public: explicit MetadataWrapper(const Scraper& scraper) : scraper_(scraper) {} - [[nodiscard]] PROMPP_LAMBDA_INLINE uint32_t size() const noexcept { return scraper_.metadata_buffer_.items_count(); } - [[nodiscard]] PROMPP_LAMBDA_INLINE auto begin() const noexcept { return scraper_.metadata_buffer_.begin(scraper_.parser_.tokenizer().buffer()); } - [[nodiscard]] PROMPP_LAMBDA_INLINE static auto end() noexcept { return MetadataMarkupBuffer::end(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t size() const noexcept { return scraper_.metadata_buffer_.items_count(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE auto begin() const noexcept { return scraper_.metadata_buffer_.begin(scraper_.parser_.tokenizer().buffer()); } + [[nodiscard]] PROMPP_ALWAYS_INLINE static auto end() noexcept { return MetadataMarkupBuffer::end(); } private: const Scraper& scraper_; }; - [[nodiscard]] PROMPP_LAMBDA_INLINE uint32_t size() const noexcept { return metric_buffer_.items_count(); } - [[nodiscard]] PROMPP_LAMBDA_INLINE auto begin() const noexcept { return metric_buffer_.begin(parser_.tokenizer().buffer(), default_timestamp_); } - [[nodiscard]] PROMPP_LAMBDA_INLINE static auto end() noexcept { return MetricMarkupBuffer::end(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t size() const noexcept { return metric_buffer_.items_count(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE auto begin() const noexcept { return metric_buffer_.begin(parser_.tokenizer().buffer(), default_timestamp_); } + [[nodiscard]] PROMPP_ALWAYS_INLINE static auto end() noexcept { return MetricMarkupBuffer::end(); } [[nodiscard]] PROMPP_ALWAYS_INLINE MetricsWrapper metrics() const noexcept { return MetricsWrapper{*this}; } [[nodiscard]] PROMPP_ALWAYS_INLINE MetadataWrapper metadata() const noexcept { return MetadataWrapper{*this}; } @@ -194,7 +194,7 @@ class Scraper { uint32_t labels_count{}; encoding::LayoutMarker layout{}; - ptr = decode_varint(ptr, layout, labels_count); + ptr = decode_count(ptr, layout, labels_count); ts.label_set().resize(labels_count); auto label_iter = ts.label_set().begin(); @@ -214,7 +214,7 @@ class Scraper { const MarkedMetric* item_{}; Primitives::Timestamp default_timestamp_; - PROMPP_ALWAYS_INLINE static const char* decode_varint(const char* ptr, encoding::LayoutMarker& layout, uint32_t& labels_count) noexcept { + PROMPP_ALWAYS_INLINE static const char* decode_count(const char* ptr, encoding::LayoutMarker& layout, uint32_t& labels_count) noexcept { uint64_t chunk; std::memcpy(&chunk, ptr, sizeof(chunk)); @@ -484,17 +484,9 @@ class Scraper { bytes_shrink(offset + bytes_written); } - void add_label(MarkedLabel label, uint32_t base_offset) noexcept { + void add_label(MarkedLabel label) noexcept { static constexpr uint8_t szm[4] = {0, 1, 2, 4}; - if (label.name.is_reserved_name()) [[unlikely]] { - label.name.offset = 0; - label.name.length = 0; - } else { - label.name.offset -= base_offset; - } - label.value.offset -= base_offset; - const uint32_t offset = bytes_count(); char* out = bytes_buffer_.data() + offset; char* start = out++; @@ -769,65 +761,40 @@ class Scraper { metric_buffer_.bytes_enlarge(bytes_offset + calculate_metric_prealloc_size(labels_.size())); metric_buffer_.bytes_shrink(bytes_offset); - const encoding::LayoutMarker layout = create_layout_marker(); + const encoding::LayoutMarker layout = + encoding::LayoutMarker::make(marked_sample_.has_ts, labels_.size(), encoding::value_type(marked_sample_.sample.value())); metric_buffer_.add_layout(layout.raw); process_labels_buffer(metric_offset); metric_buffer_.add_sample(layout, marked_sample_.sample); } - [[nodiscard]] encoding::LayoutMarker create_layout_marker() noexcept { - using encoding::LayoutMarker; - using encoding::SampleValueType; - - const uint32_t labels_count = labels_.size(); - const double val = marked_sample_.sample.value(); - - if (std::isnan(val)) [[unlikely]] { - return LayoutMarker::make(marked_sample_.has_ts, labels_count, SampleValueType::kNaN); - } - if (val == 0.0) [[unlikely]] { - return LayoutMarker::make(marked_sample_.has_ts, labels_count, SampleValueType::kZero); - } - - if (std::trunc(val) == val && val > 0.0) [[likely]] { - const auto uval = static_cast(val); - if (uval <= std::numeric_limits::max()) { - return LayoutMarker::make(marked_sample_.has_ts, labels_count, SampleValueType::kUint8); - } - if (uval <= std::numeric_limits::max()) { - return LayoutMarker::make(marked_sample_.has_ts, labels_count, SampleValueType::kUint16); - } - if (uval <= std::numeric_limits::max()) { - return LayoutMarker::make(marked_sample_.has_ts, labels_count, SampleValueType::kUint32); - } - } - - const auto f = static_cast(val); - if (static_cast(f) == val) [[unlikely]] { - return LayoutMarker::make(marked_sample_.has_ts, labels_count, SampleValueType::kFloat); - } - - return LayoutMarker::make(marked_sample_.has_ts, labels_count, SampleValueType::kDouble); - } - void process_labels_buffer(uint32_t offset) noexcept { sort_and_filter_labels(); append_labels_hash(); metric_buffer_.add_count(labels_.size()); - for (const auto& label : labels_) { - metric_buffer_.add_label(label, offset); + for (auto& label : labels_) { + if (label.name.is_reserved_name()) [[unlikely]] { + label.name.offset = 0; + label.name.length = 0; + } else { + label.name.offset -= offset; + } + label.value.offset -= offset; + + metric_buffer_.add_label(label); } } void sort_and_filter_labels() noexcept { - const auto it = std::remove_if(labels_.begin(), labels_.end(), [](const MarkedLabel& label) { return label.value.is_empty(); }); + const auto it = std::remove_if(labels_.begin(), labels_.end(), [](const MarkedLabel& label) PROMPP_LAMBDA_INLINE { return label.value.is_empty(); }); labels_.erase(it, labels_.end()); - std::sort(labels_.begin(), labels_.end(), - [buffer = parser_.tokenizer().buffer()](const MarkedLabel& a, const MarkedLabel& b) { return a.name.view(buffer) < b.name.view(buffer); }); + std::sort(labels_.begin(), labels_.end(), [buffer = parser_.tokenizer().buffer()](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { + return a.name.view(buffer) < b.name.view(buffer); + }); } void append_labels_hash() noexcept { @@ -840,11 +807,11 @@ class Scraper { } static PROMPP_ALWAYS_INLINE uint32_t calculate_metric_prealloc_size(uint32_t labels_count) noexcept { - constexpr uint32_t kCountVarintPreallocBytes = 8; - constexpr uint32_t kLabelPreallocBytes = 17; - constexpr uint32_t kSamplePreallocSize = 17; + constexpr uint32_t kCountVarintBytes = 8; + constexpr uint32_t kLabelBytes = 17; + constexpr uint32_t kSampleSize = 16; - return kCountVarintPreallocBytes + labels_count * kLabelPreallocBytes + kSamplePreallocSize; + return kCountVarintBytes + labels_count * kLabelBytes + kSampleSize; } Parser parser_; From 86b5b041d4046600d0f90679e7ded306653e3ae2 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 10 Sep 2025 19:45:09 +0300 Subject: [PATCH 28/48] LabelCodec + tests --- pp/wal/hashdex/scraper/encoding.h | 160 +++++++++++++++++++++ pp/wal/hashdex/scraper/encoding_tests.cpp | 147 +++++++++++++++++++ pp/wal/hashdex/scraper/scraper.h | 164 ++-------------------- 3 files changed, 319 insertions(+), 152 deletions(-) diff --git a/pp/wal/hashdex/scraper/encoding.h b/pp/wal/hashdex/scraper/encoding.h index 8d2d0fa540..cfbd4b1ca1 100644 --- a/pp/wal/hashdex/scraper/encoding.h +++ b/pp/wal/hashdex/scraper/encoding.h @@ -134,4 +134,164 @@ class SampleCodec { } }; +class LabelCodec { + public: + static PROMPP_ALWAYS_INLINE char* encode(char* out, + const uint32_t label_name_offset, + const uint32_t label_name_length, + const uint32_t label_value_offset, + const uint32_t label_value_length) noexcept { + static constexpr uint8_t szm[4] = {0, 1, 2, 4}; + + char* start = out++; + const uint8_t sz0 = push_and_encode(out, label_name_offset); + out += szm[sz0]; + const uint8_t sz1 = push_and_encode(out, label_name_length); + out += szm[sz1]; + const uint8_t sz2 = push_and_encode(out, label_value_offset); + out += szm[sz2]; + const uint8_t sz3 = push_and_encode(out, label_value_length); + out += szm[sz3]; + + *start = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); + + return out; + } + + struct DecodeResult { + const char* next; + uint32_t label_name_offset; + uint32_t label_name_length; + uint32_t label_value_offset; + uint32_t label_value_length; + }; + + static PROMPP_ALWAYS_INLINE DecodeResult decode(const char* in) noexcept { + const char* start = in; + + uint64_t chunk; + std::memcpy(&chunk, in, sizeof(chunk)); + const auto layout = static_cast(chunk); + + if (layout == 0b01010101) [[likely]] { + const auto name_off = static_cast(chunk >> 8); + const auto name_len = static_cast(chunk >> 16); + const auto value_off = static_cast(chunk >> 24); + const auto value_len = static_cast(chunk >> 32); + + return DecodeResult{ + .next = start + 5, .label_name_offset = name_off, .label_name_length = name_len, .label_value_offset = value_off, .label_value_length = value_len}; + } + + if ((layout & 0x0F) == 0) [[likely]] { + chunk >>= 8; + + const uint8_t sz2 = (layout >> 4) & 0x3; + const uint8_t sz3 = (layout >> 6) & 0x3; + + uint32_t value_off = 0; + uint32_t value_len = 0; + size_t used = 0; + + // value_off + switch (sz2) { + case 0b00: + break; + case 0b01: + value_off = static_cast(chunk); + used += 1; + chunk >>= 8; + break; + case 0b10: + value_off = static_cast(chunk); + used += 2; + chunk >>= 16; + break; + default: + value_off = static_cast(chunk); + used += 4; + chunk >>= 32; + break; + } + + // value_len + switch (sz3) { + case 0b00: + break; + case 0b01: + value_len = static_cast(chunk); + used += 1; + break; + case 0b10: + value_len = static_cast(chunk); + used += 2; + break; + default: + + if (used + 4 <= 7) [[likely]] { + value_len = static_cast(chunk); + } else [[unlikely]] { + std::memcpy(&value_len, start + 1 + used, 4); + } + used += 4; + break; + } + + return DecodeResult{ + .next = start + 1 + used, .label_name_offset = 0, .label_name_length = 0, .label_value_offset = value_off, .label_value_length = value_len}; + } + + in += 1; + const uint8_t sz0 = (layout >> 0) & 0x3; + const uint8_t sz1 = (layout >> 2) & 0x3; + const uint8_t sz2 = (layout >> 4) & 0x3; + const uint8_t sz3 = (layout >> 6) & 0x3; + + const uint32_t name_off = read_val_partial(in, sz0); + const uint32_t name_len = read_val_partial(in, sz1); + const uint32_t value_off = read_val_partial(in, sz2); + const uint32_t value_len = read_val_partial(in, sz3); + + return DecodeResult{ + .next = in, .label_name_offset = name_off, .label_name_length = name_len, .label_value_offset = value_off, .label_value_length = value_len}; + } + + private: + static PROMPP_ALWAYS_INLINE uint8_t push_and_encode(char* out, uint32_t v) noexcept { + if (v == 0) [[unlikely]] { + return 0b00; + } + if (v <= 0xFF) [[likely]] { + *out = static_cast(v); + return 0b01; + } + if (v <= 0xFFFF) [[unlikely]] { + const auto value = static_cast(v); + std::memcpy(out, &value, sizeof(value)); + return 0b10; + } + std::memcpy(out, &v, sizeof(v)); + return 0b11; + } + + static PROMPP_ALWAYS_INLINE uint32_t read_val_partial(const char*& p, uint8_t sz) noexcept { + if (sz == 0b01) [[likely]] { + return *p++; + } + if (sz == 0b00) { + return 0; + } + if (sz == 0b10) { + uint16_t v; + std::memcpy(&v, p, 2); + p += 2; + return v; + } + uint32_t v; + std::memcpy(&v, p, 4); + p += 4; + return v; + } +}; + } // namespace PromPP::WAL::hashdex::scraper::encoding \ No newline at end of file diff --git a/pp/wal/hashdex/scraper/encoding_tests.cpp b/pp/wal/hashdex/scraper/encoding_tests.cpp index 830361f109..0a33636232 100644 --- a/pp/wal/hashdex/scraper/encoding_tests.cpp +++ b/pp/wal/hashdex/scraper/encoding_tests.cpp @@ -232,4 +232,151 @@ TEST_F(SampleCodecFixture, Decode_ReturnedPointerAdvancesCorrectly_WithTimestamp EXPECT_EQ(res.sample.timestamp(), 2048); } +class LabelCodecFixture : public testing::Test { + protected: + static constexpr size_t kBufSize = 64; + std::array buf_{}; + + LabelCodec::DecodeResult encode_and_decode(uint32_t name_off, uint32_t name_len, uint32_t value_off, uint32_t value_len) { + buf_.fill(0); + char* start = buf_.data(); + char* end = LabelCodec::encode(start, name_off, name_len, value_off, value_len); + + const auto res = LabelCodec::decode(start); + + EXPECT_EQ(res.next, end); + return res; + } +}; + +TEST_F(LabelCodecFixture, EncodeDecode_AllZeros) { + // Arrange + + // Act + const auto res = encode_and_decode(0, 0, 0, 0); + + // Assert + EXPECT_EQ(res.label_name_offset, 0u); + EXPECT_EQ(res.label_name_length, 0u); + EXPECT_EQ(res.label_value_offset, 0u); + EXPECT_EQ(res.label_value_length, 0u); +} + +TEST_F(LabelCodecFixture, EncodeDecode_OneByteValues) { + // Arrange + + // Act + const auto res = encode_and_decode(1, 2, 3, 4); + + EXPECT_EQ(res.label_name_offset, 1u); + EXPECT_EQ(res.label_name_length, 2u); + EXPECT_EQ(res.label_value_offset, 3u); + EXPECT_EQ(res.label_value_length, 4u); +} + +TEST_F(LabelCodecFixture, EncodeDecode_TwoByteValues) { + // Arrange + + // Act + const auto res = encode_and_decode(300, 400, 500, 600); + + EXPECT_EQ(res.label_name_offset, 300u); + EXPECT_EQ(res.label_name_length, 400u); + EXPECT_EQ(res.label_value_offset, 500u); + EXPECT_EQ(res.label_value_length, 600u); +} + +TEST_F(LabelCodecFixture, EncodeDecode_FourByteValues) { + // Arrange + + // Act + const auto res = encode_and_decode(100000, 200000, 300000, 400000); + + // Assert + EXPECT_EQ(res.label_name_offset, 100000u); + EXPECT_EQ(res.label_name_length, 200000u); + EXPECT_EQ(res.label_value_offset, 300000u); + EXPECT_EQ(res.label_value_length, 400000u); +} + +TEST_F(LabelCodecFixture, EncodeDecode_0044) { + // Arrange + + // Act + const auto res = encode_and_decode(0, 0, 300000, 400000); + + // Assert + EXPECT_EQ(res.label_name_offset, 0); + EXPECT_EQ(res.label_name_length, 0); + EXPECT_EQ(res.label_value_offset, 300000u); + EXPECT_EQ(res.label_value_length, 400000u); +} + +TEST_F(LabelCodecFixture, EncodeDecode_0024) { + // Arrange + + // Act + const auto res = encode_and_decode(0, 0, 1234, 123456); + + // Assert + EXPECT_EQ(res.label_name_offset, 0); + EXPECT_EQ(res.label_name_length, 0); + EXPECT_EQ(res.label_value_offset, 1234); + EXPECT_EQ(res.label_value_length, 123456); +} + +TEST_F(LabelCodecFixture, EncodeDecode_0124) { + // Arrange + + // Act + const auto res = encode_and_decode(0, 12, 1234, 123456); + + // Assert + EXPECT_EQ(res.label_name_offset, 0); + EXPECT_EQ(res.label_name_length, 12); + EXPECT_EQ(res.label_value_offset, 1234); + EXPECT_EQ(res.label_value_length, 123456); +} + +TEST_F(LabelCodecFixture, FastPath_Layout01010101) { + // Arrange + buf_.fill(0); + char* start = buf_.data(); + start[0] = static_cast(0b01010101); + start[1] = 10; + start[2] = 11; + start[3] = 12; + start[4] = 13; + + // Act + const auto res = LabelCodec::decode(buf_.data()); + + // Assert + EXPECT_EQ(res.label_name_offset, 10u); + EXPECT_EQ(res.label_name_length, 11u); + EXPECT_EQ(res.label_value_offset, 12u); + EXPECT_EQ(res.label_value_length, 13u); + EXPECT_EQ(res.next, buf_.data() + 5); +} + +TEST_F(LabelCodecFixture, SimplifiedPath_NameFieldsZero) { + // Arrange + buf_.fill(0); + uint8_t layout = 0b10010000; + buf_[0] = static_cast(layout); + buf_[1] = 42; + uint16_t len = 1234; + std::memcpy(buf_.data() + 2, &len, 2); + + // Act + const auto res = LabelCodec::decode(buf_.data()); + + // Assert + EXPECT_EQ(res.label_name_offset, 0u); + EXPECT_EQ(res.label_name_length, 0u); + EXPECT_EQ(res.label_value_offset, 42u); + EXPECT_EQ(res.label_value_length, 1234u); + EXPECT_EQ(res.next, buf_.data() + 1 + 1 + 2); +} + } // namespace \ No newline at end of file diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 5a22d5cfa4..fd131f9d12 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -189,7 +189,6 @@ class Scraper { template void read(Timeseries& ts) const { const char* ptr = reinterpret_cast(bytes_buffer_.data() + item_->data_offset); - const char* base = buffer_.data() + item_->base_offset; uint32_t labels_count{}; encoding::LayoutMarker layout{}; @@ -198,9 +197,16 @@ class Scraper { ts.label_set().resize(labels_count); auto label_iter = ts.label_set().begin(); + const char* base = buffer_.data() + item_->base_offset; for (uint32_t i = 0; i < labels_count; ++i) { - const uint32_t consumed_bytes = decode_label(ptr, base, label_iter++); - ptr += consumed_bytes; + const auto [next_ptr, name_off, name_len, value_off, value_len] = encoding::LabelCodec::decode(ptr); + ptr = next_ptr; + + if (name_len == 0 && name_off == 0) [[unlikely]] { + std::construct_at(label_iter++, Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); + } else { + std::construct_at(label_iter++, std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); + } } auto [p, sample] = encoding::SampleCodec::decode(ptr, layout, default_timestamp_); @@ -227,136 +233,6 @@ class Scraper { return ptr + 1 + layout.count_size_in_bytes(); } - - PROMPP_ALWAYS_INLINE static uint32_t read_val_partial(const char*& p, uint8_t sz) noexcept { - if (sz == 0b01) [[likely]] { - return *p++; - } - - if (sz == 0b00) { - return 0; - } - - if (sz == 0b10) { - uint16_t v; - std::memcpy(&v, p, 2); - p += 2; - return v; - } - - uint32_t v; - std::memcpy(&v, p, 4); - p += 4; - return v; - } - - template - PROMPP_ALWAYS_INLINE static uint32_t decode_label(const char* ptr, const char* base, LabelSetIter iter) noexcept { - const char* start = ptr; - - uint64_t chunk; - std::memcpy(&chunk, ptr, sizeof(chunk)); - uint8_t layout = static_cast(chunk); - - if (layout == 0b01010101) [[likely]] { - const uint8_t name_off = static_cast(chunk >> 8); - const uint8_t name_len = static_cast(chunk >> 16); - const uint8_t value_off = static_cast(chunk >> 24); - const uint8_t value_len = static_cast(chunk >> 32); - - std::construct_at(iter, std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); - - return 5; - } - - if ((layout & 0x0F) == 0) [[likely]] { - chunk >>= 8; - - const uint8_t sz2 = (layout >> 4) & 0x3; - const uint8_t sz3 = (layout >> 6) & 0x3; - - uint32_t value_off = 0; - uint32_t value_len = 0; - size_t used = 0; - - // value_off - switch (sz2) { - case 0b00: { - value_off = 0; - break; - } - case 0b01: { - value_off = static_cast(chunk); - used += 1; - chunk >>= 8; - break; - } - case 0b10: { - value_off = static_cast(chunk); - used += 2; - chunk >>= 16; - break; - } - default: { - value_off = static_cast(chunk); - used += 4; - chunk >>= 32; - break; - } - } - - // value_len - switch (sz3) { - case 0b00: { - value_len = 0; - break; - } - case 0b01: { - value_len = static_cast(chunk); - used += 1; - break; - } - case 0b10: { - value_len = static_cast(chunk); - used += 2; - break; - } - default: { - used += 4; - if (used <= 7) [[likely]] { - value_len = static_cast(chunk); - } else [[unlikely]] { - std::memcpy(&value_len, start + 1 + used, 4); - } - break; - } - } - - std::construct_at(iter, Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); - - return 1 + used; - } - - ptr += 1; - - const uint8_t sz0 = (layout >> 0) & 0x3; - const uint8_t sz1 = (layout >> 2) & 0x3; - const uint8_t sz2 = (layout >> 4) & 0x3; - const uint8_t sz3 = (layout >> 6) & 0x3; - - const uint32_t name_off = read_val_partial(ptr, sz0); - const uint32_t name_len = read_val_partial(ptr, sz1); - const uint32_t value_off = read_val_partial(ptr, sz2); - const uint32_t value_len = read_val_partial(ptr, sz3); - - if (name_len == 0 && name_off == 0) [[unlikely]] { - std::construct_at(iter, Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); - } else { - std::construct_at(iter, std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); - } - - return static_cast(ptr - start); - } }; class Metadata { @@ -485,25 +361,9 @@ class Scraper { } void add_label(MarkedLabel label) noexcept { - static constexpr uint8_t szm[4] = {0, 1, 2, 4}; - - const uint32_t offset = bytes_count(); - char* out = bytes_buffer_.data() + offset; - char* start = out++; - - const uint8_t sz0 = push_and_encode(out, label.name.offset); - out += szm[sz0]; - const uint8_t sz1 = push_and_encode(out, label.name.length); - out += szm[sz1]; - const uint8_t sz2 = push_and_encode(out, label.value.offset); - out += szm[sz2]; - const uint8_t sz3 = push_and_encode(out, label.value.length); - out += szm[sz3]; - - *start = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); - - const auto written = static_cast(out - start); - bytes_shrink(offset + written); + const auto end = + encoding::LabelCodec::encode(bytes_buffer_.data() + bytes_count(), label.name.offset, label.name.length, label.value.offset, label.value.length); + bytes_shrink(end - bytes_buffer_.data()); } void add_sample(encoding::LayoutMarker layout, const Primitives::Sample& sample) noexcept { From d749eb99ad6b9aa4cc0f011ede85c798e5ebea7e Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 10 Sep 2025 20:09:51 +0300 Subject: [PATCH 29/48] fixes --- pp/wal/hashdex/scraper/scraper.h | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index fd131f9d12..b50d70f66c 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -119,8 +119,8 @@ class Scraper { #pragma pack(push, 1) struct MarkedString { - uint32_t offset{std::numeric_limits::max()}; - uint32_t length{std::numeric_limits::max()}; + uint32_t offset = 0; + uint32_t length = 0; [[nodiscard]] PROMPP_ALWAYS_INLINE static MarkedString create(std::string_view value, std::string_view buffer) noexcept { return { @@ -129,9 +129,7 @@ class Scraper { }; } - [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_reserved_name() const noexcept { - return offset == std::numeric_limits::max() && length == std::numeric_limits::max(); - } + [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_reserved_name() const noexcept { return offset == 0 && length == 0; } [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_empty() const noexcept { return length == 0; } @@ -383,23 +381,6 @@ class Scraper { private: BareBones::Vector bytes_buffer_{}; uint32_t bytes_count_ = 0; - - PROMPP_ALWAYS_INLINE static uint8_t push_and_encode(char* out, uint32_t v) noexcept { - if (v == 0) [[unlikely]] { - return 0b00; - } - if (v <= 0xFF) [[likely]] { - *out = static_cast(v); - return 0b01; - } - if (v <= 0xFFFF) [[unlikely]] { - const auto value = static_cast(v); - std::memcpy(out, &value, sizeof(value)); - return 0b10; - } - std::memcpy(out, &v, sizeof(v)); - return 0b11; - } }; class MetadataMarkupBuffer : public MarkupBuffer { @@ -498,7 +479,7 @@ class Scraper { return Error::kNoMetricName; } - auto error = parse_metric_suffix(); + const auto error = parse_metric_suffix(); if (error == Error::kNoError) [[likely]] { encode_metric_data(metric_offset); @@ -636,10 +617,7 @@ class Scraper { metric_buffer_.add_count(labels_.size()); for (auto& label : labels_) { - if (label.name.is_reserved_name()) [[unlikely]] { - label.name.offset = 0; - label.name.length = 0; - } else { + if (!label.name.is_reserved_name()) [[likely]] { label.name.offset -= offset; } label.value.offset -= offset; From cb9446a2a9fb34a1ca9786053874c44412e08baa Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 10 Sep 2025 23:03:29 +0300 Subject: [PATCH 30/48] refactoring --- pp/wal/hashdex/scraper/encoding.h | 85 ++++----- pp/wal/hashdex/scraper/marked.h | 302 ++++++++++++++++++++++++++++++ pp/wal/hashdex/scraper/scraper.h | 292 +---------------------------- 3 files changed, 336 insertions(+), 343 deletions(-) create mode 100644 pp/wal/hashdex/scraper/marked.h diff --git a/pp/wal/hashdex/scraper/encoding.h b/pp/wal/hashdex/scraper/encoding.h index cfbd4b1ca1..222d9b7aae 100644 --- a/pp/wal/hashdex/scraper/encoding.h +++ b/pp/wal/hashdex/scraper/encoding.h @@ -4,6 +4,7 @@ #include #include +#include "bare_bones/bit.h" #include "primitives/sample.h" #include "prometheus/value.h" @@ -183,69 +184,49 @@ class LabelCodec { .next = start + 5, .label_name_offset = name_off, .label_name_length = name_len, .label_value_offset = value_off, .label_value_length = value_len}; } + const uint8_t sz0 = layout & 0b11; + const uint8_t sz1 = (layout >> 2) & 0b11; + const uint8_t sz2 = (layout >> 4) & 0b11; + const uint8_t sz3 = (layout >> 6) & 0b11; + if ((layout & 0x0F) == 0) [[likely]] { chunk >>= 8; - - const uint8_t sz2 = (layout >> 4) & 0x3; - const uint8_t sz3 = (layout >> 6) & 0x3; - + size_t used = 1; uint32_t value_off = 0; uint32_t value_len = 0; - size_t used = 0; - - // value_off - switch (sz2) { - case 0b00: - break; - case 0b01: - value_off = static_cast(chunk); - used += 1; - chunk >>= 8; - break; - case 0b10: - value_off = static_cast(chunk); - used += 2; - chunk >>= 16; - break; - default: - value_off = static_cast(chunk); - used += 4; - chunk >>= 32; - break; + if (sz2 == 0b01) [[likely]] { + value_off = static_cast(chunk); + used += sizeof(uint8_t); + chunk >>= BareBones::Bit::to_bits(sizeof(uint8_t)); + } else if (sz2 == 0b10) [[unlikely]] { + value_off = static_cast(chunk); + used += sizeof(uint16_t); + chunk >>= BareBones::Bit::to_bits(sizeof(uint16_t)); + } else if (sz2 == 0b11) [[unlikely]] { + value_off = static_cast(chunk); + used += sizeof(uint32_t); + chunk >>= BareBones::Bit::to_bits(sizeof(uint32_t)); } - - // value_len - switch (sz3) { - case 0b00: - break; - case 0b01: - value_len = static_cast(chunk); - used += 1; - break; - case 0b10: - value_len = static_cast(chunk); - used += 2; - break; - default: - - if (used + 4 <= 7) [[likely]] { - value_len = static_cast(chunk); - } else [[unlikely]] { - std::memcpy(&value_len, start + 1 + used, 4); - } - used += 4; - break; + if (sz3 == 0b01) [[likely]] { + value_len = static_cast(chunk); + used += sizeof(uint8_t); + } else if (sz3 == 0b10) [[unlikely]] { + value_len = static_cast(chunk); + used += sizeof(uint16_t); + } else if (sz3 == 0b11) [[unlikely]] { + if (used + sizeof(uint32_t) <= sizeof(chunk)) [[likely]] { + value_len = static_cast(chunk); + } else [[unlikely]] { + std::memcpy(&value_len, start + used, sizeof(value_len)); + } + used += sizeof(value_len); } return DecodeResult{ - .next = start + 1 + used, .label_name_offset = 0, .label_name_length = 0, .label_value_offset = value_off, .label_value_length = value_len}; + .next = start + used, .label_name_offset = 0, .label_name_length = 0, .label_value_offset = value_off, .label_value_length = value_len}; } in += 1; - const uint8_t sz0 = (layout >> 0) & 0x3; - const uint8_t sz1 = (layout >> 2) & 0x3; - const uint8_t sz2 = (layout >> 4) & 0x3; - const uint8_t sz3 = (layout >> 6) & 0x3; const uint32_t name_off = read_val_partial(in, sz0); const uint32_t name_len = read_val_partial(in, sz1); diff --git a/pp/wal/hashdex/scraper/marked.h b/pp/wal/hashdex/scraper/marked.h new file mode 100644 index 0000000000..b1b8ad3c73 --- /dev/null +++ b/pp/wal/hashdex/scraper/marked.h @@ -0,0 +1,302 @@ +#pragma once + +#include +#include + +#include "bare_bones/bit.h" +#include "bare_bones/vector.h" +#include "encoding.h" +#include "primitives/primitives.h" +#include "prometheus/metric.h" +#include "prometheus/value.h" + +namespace PromPP::WAL::hashdex::scraper::inline marked { + +#pragma pack(push, 1) +struct MarkedString { + uint32_t offset = 0; + uint32_t length = 0; + + [[nodiscard]] PROMPP_ALWAYS_INLINE static MarkedString create(std::string_view value, std::string_view buffer) noexcept { + return { + .offset = static_cast(value.data() - buffer.data()), + .length = static_cast(value.size()), + }; + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_reserved_name() const noexcept { return offset == 0 && length == 0; } + + [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_empty() const noexcept { return length == 0; } + + [[nodiscard]] std::string_view view(const std::string_view& buffer) const noexcept { + if (is_reserved_name()) [[unlikely]] { + return Prometheus::kMetricLabelName; + } + + return buffer.substr(offset, length); + } +}; + +struct MarkedLabel { + MarkedString name{}; + MarkedString value; +}; + +struct MarkedSample { + Primitives::Sample sample{}; + bool has_ts{}; +}; + +struct MarkedMetric { + uint64_t hash; + uint32_t base_offset; + uint32_t data_offset; +}; + +struct MarkedMetadata { + MarkedString metric_name{}; + MarkedString text{}; + Prometheus::MetadataType type{}; +}; +#pragma pack(pop) + +class Metric { + public: + using MarkedT = MarkedMetric; + + struct Context { + std::string_view buffer; + const BareBones::Vector& bytes_buffer; + Primitives::Timestamp default_timestamp{}; + }; + + Metric(const Context& ctx, const MarkedMetric* item) + : buffer_(ctx.buffer), bytes_buffer_(ctx.bytes_buffer), item_(item), default_timestamp_(ctx.default_timestamp) {} + + [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetric* item() const noexcept { return item_; } + PROMPP_ALWAYS_INLINE void set_item(const MarkedMetric* item) noexcept { item_ = item; } + + [[nodiscard]] PROMPP_ALWAYS_INLINE uint64_t hash() const noexcept { return item_->hash; } + + template + void read(Timeseries& ts) const { + const char* ptr = reinterpret_cast(bytes_buffer_.data() + item_->data_offset); + + uint32_t labels_count{}; + encoding::LayoutMarker layout{}; + + ptr = decode_count(ptr, layout, labels_count); + ts.label_set().resize(labels_count); + + auto label_iter = ts.label_set().begin(); + const char* base = buffer_.data() + item_->base_offset; + for (uint32_t i = 0; i < labels_count; ++i) { + const auto [next_ptr, name_off, name_len, value_off, value_len] = encoding::LabelCodec::decode(ptr); + ptr = next_ptr; + + if (name_len == 0 && name_off == 0) [[unlikely]] { + std::construct_at(label_iter++, Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); + } else { + std::construct_at(label_iter++, std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); + } + } + + auto [p, sample] = encoding::SampleCodec::decode(ptr, layout, default_timestamp_); + + ts.samples().emplace_back(sample); + } + + private: + std::string_view buffer_; + const BareBones::Vector& bytes_buffer_; + const MarkedMetric* item_{}; + Primitives::Timestamp default_timestamp_; + + PROMPP_ALWAYS_INLINE static const char* decode_count(const char* ptr, encoding::LayoutMarker& layout, uint32_t& labels_count) noexcept { + uint64_t chunk; + std::memcpy(&chunk, ptr, sizeof(chunk)); + + layout = encoding::LayoutMarker{.raw = static_cast(chunk)}; + + chunk >>= 8; + const uint64_t mask = (1ULL << BareBones::Bit::to_bits(layout.count_size_in_bytes())) - 1; + + labels_count = static_cast(chunk & mask); + + return ptr + 1 + layout.count_size_in_bytes(); + } +}; + +class Metadata { + public: + using MarkedT = MarkedMetadata; + + struct Context { + std::string_view buffer; + }; + + Metadata(const Context& ctx, const MarkedMetadata* item) : buffer_(ctx.buffer), item_(item) {} + + [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetadata* item() const noexcept { return item_; } + PROMPP_ALWAYS_INLINE void set_item(const MarkedMetadata* item) noexcept { item_ = item; } + + [[nodiscard]] PROMPP_ALWAYS_INLINE Prometheus::MetadataType type() const noexcept { return item_->type; } + [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view metric_name() const noexcept { return item_->metric_name.view(buffer_); } + [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view text() const noexcept { return item_->text.view(buffer_); } + + private: + std::string_view buffer_; + const MarkedMetadata* item_{}; +}; + +template +class MarkupBuffer { + public: + using MarkedT = typename T::MarkedT; + using Context = typename T::Context; + + class IteratorSentinel {}; + + class Iterator { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + Iterator(const Context& ctx, const MarkedT* ptr, uint32_t items_count) : item_(ctx, ptr), ptr_(ptr), items_count_(items_count), ctx_(ctx) {} + + [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type& operator*() const noexcept { return item_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type* operator->() const noexcept { return &item_; } + + PROMPP_ALWAYS_INLINE Iterator& operator++() noexcept { + item_.set_item(++ptr_); + --items_count_; + return *this; + } + + PROMPP_ALWAYS_INLINE Iterator operator++(int) noexcept { + auto tmp = *this; + ++(*this); + return tmp; + } + + PROMPP_ALWAYS_INLINE bool operator==(const IteratorSentinel&) const noexcept { return items_count_ == 0; } + + private: + T item_; + const MarkedT* ptr_; + uint32_t items_count_; + Context ctx_; + }; + + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t items_count() const noexcept { return buffer_.size(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return buffer_.allocated_memory(); } + + protected: + BareBones::Vector buffer_; +}; + +class MetricMarkupBuffer : public MarkupBuffer { + public: + using Base = MarkupBuffer; + using Iterator = typename Base::Iterator; + using IteratorSentinel = typename Base::IteratorSentinel; + + [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer, Primitives::Timestamp default_ts) const noexcept { + return {typename Base::Context{buffer, bytes_buffer_, default_ts}, this->buffer_.data(), this->items_count()}; + } + [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } + + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t bytes_count() const noexcept { return bytes_count_; } + void bytes_enlarge(uint32_t new_size) noexcept { + assert(new_size > bytes_count_); + if (new_size > bytes_buffer_.size()) [[likely]] { + bytes_buffer_.resize(new_size); + } + bytes_count_ = new_size; + } + + void bytes_shrink(uint32_t new_size) noexcept { bytes_count_ = new_size; } + + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return this->buffer_.allocated_memory() + bytes_buffer_.allocated_memory(); } + + PROMPP_ALWAYS_INLINE void initialize(size_t reserve_bytes) noexcept { + this->buffer_.clear(); + + const size_t bytes_buffer_reserve = (reserve_bytes / 3) * 2; + const size_t items_buffer_reserve = (bytes_buffer_reserve / 3) / sizeof(MarkedMetric); + + this->buffer_.reserve(items_buffer_reserve); + bytes_buffer_.reserve(bytes_buffer_reserve); + bytes_count_ = 0; + } + + PROMPP_ALWAYS_INLINE void add_hash(uint64_t hash) noexcept { this->buffer_.back().hash = hash; } + + PROMPP_ALWAYS_INLINE void add_metric(uint32_t global_offset) noexcept { + this->buffer_.push_back(MarkedMetric{.hash = {}, .base_offset = global_offset, .data_offset = bytes_count()}); + } + + void add_layout(uint8_t layout) noexcept { + const uint32_t offset = bytes_count(); + *(bytes_buffer_.data() + offset) = layout; + bytes_shrink(offset + 1); + } + + void add_count(uint32_t count) noexcept { + const uint32_t bytes_written = (std::bit_width(count) + 7) / 8; + const uint32_t offset = bytes_count(); + std::memcpy(bytes_buffer_.data() + offset, &count, sizeof(count)); + bytes_shrink(offset + bytes_written); + } + + void add_label(MarkedLabel label) noexcept { + const auto end = + encoding::LabelCodec::encode(bytes_buffer_.data() + bytes_count(), label.name.offset, label.name.length, label.value.offset, label.value.length); + bytes_shrink(end - bytes_buffer_.data()); + } + + void add_sample(encoding::LayoutMarker layout, const Primitives::Sample& sample) noexcept { + using encoding::SampleValueType; + + char* data_ptr = bytes_buffer_.data(); + const char* end = encoding::SampleCodec::encode(data_ptr + bytes_count(), layout, sample); + + bytes_shrink(end - data_ptr); + } + + void add_padding() noexcept { + constexpr size_t kPaddingSizeBytes = 16; + bytes_enlarge(bytes_count() + kPaddingSizeBytes); + } + + private: + BareBones::Vector bytes_buffer_{}; + uint32_t bytes_count_ = 0; +}; + +class MetadataMarkupBuffer : public MarkupBuffer { + public: + using Base = MarkupBuffer; + using Iterator = typename Base::Iterator; + using IteratorSentinel = typename Base::IteratorSentinel; + + [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer) const noexcept { + return {typename Base::Context{buffer}, this->buffer_.data(), this->items_count()}; + } + [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } + + PROMPP_ALWAYS_INLINE void initialize(size_t reserve_bytes) noexcept { + this->buffer_.clear(); + const size_t items_buffer_reserve = reserve_bytes / sizeof(MarkedMetric); + this->buffer_.reserve(items_buffer_reserve); + } + + PROMPP_ALWAYS_INLINE void add(MarkedString metric_name, MarkedString text, Prometheus::MetadataType type) noexcept { + this->buffer_.emplace_back(metric_name, text, type); + } +}; + +} // namespace PromPP::WAL::hashdex::scraper::inline marked \ No newline at end of file diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index b50d70f66c..c92eb0ff1b 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -5,18 +5,16 @@ #include #include "bare_bones/algorithm.h" -#include "bare_bones/bit.h" #include "bare_bones/vector.h" #include "bare_bones/xxhash.h" #include "encoding.h" +#include "marked.h" #include "parser.h" #include "prometheus/hashdex.h" #include "prometheus/metric.h" #include "prometheus/textparse/escape.h" #include "prometheus/value.h" -#include "primitives/sample.h" - namespace PromPP::WAL::hashdex::scraper { template @@ -117,294 +115,6 @@ class Scraper { private: using Token = Prometheus::textparse::Token; -#pragma pack(push, 1) - struct MarkedString { - uint32_t offset = 0; - uint32_t length = 0; - - [[nodiscard]] PROMPP_ALWAYS_INLINE static MarkedString create(std::string_view value, std::string_view buffer) noexcept { - return { - .offset = static_cast(value.data() - buffer.data()), - .length = static_cast(value.size()), - }; - } - - [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_reserved_name() const noexcept { return offset == 0 && length == 0; } - - [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_empty() const noexcept { return length == 0; } - - [[nodiscard]] std::string_view view(const std::string_view& buffer) const noexcept { - if (is_reserved_name()) [[unlikely]] { - return Prometheus::kMetricLabelName; - } - - return buffer.substr(offset, length); - } - }; - - struct MarkedLabel { - MarkedString name{}; - MarkedString value; - }; - - struct MarkedSample { - Primitives::Sample sample{}; - bool has_ts{}; - }; - - struct MarkedMetric { - uint64_t hash; - uint32_t base_offset; - uint32_t data_offset; - }; - - struct MarkedMetadata { - MarkedString metric_name{}; - MarkedString text{}; - Prometheus::MetadataType type{}; - }; -#pragma pack(pop) - - public: - class Metric { - public: - using MarkedT = MarkedMetric; - - struct Context { - std::string_view buffer; - const BareBones::Vector& bytes_buffer; - Primitives::Timestamp default_timestamp{}; - }; - - Metric(const Context& ctx, const MarkedMetric* item) - : buffer_(ctx.buffer), bytes_buffer_(ctx.bytes_buffer), item_(item), default_timestamp_(ctx.default_timestamp) {} - - [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetric* item() const noexcept { return item_; } - PROMPP_ALWAYS_INLINE void set_item(const MarkedMetric* item) noexcept { item_ = item; } - - [[nodiscard]] PROMPP_ALWAYS_INLINE uint64_t hash() const noexcept { return item_->hash; } - - template - void read(Timeseries& ts) const { - const char* ptr = reinterpret_cast(bytes_buffer_.data() + item_->data_offset); - - uint32_t labels_count{}; - encoding::LayoutMarker layout{}; - - ptr = decode_count(ptr, layout, labels_count); - ts.label_set().resize(labels_count); - - auto label_iter = ts.label_set().begin(); - const char* base = buffer_.data() + item_->base_offset; - for (uint32_t i = 0; i < labels_count; ++i) { - const auto [next_ptr, name_off, name_len, value_off, value_len] = encoding::LabelCodec::decode(ptr); - ptr = next_ptr; - - if (name_len == 0 && name_off == 0) [[unlikely]] { - std::construct_at(label_iter++, Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); - } else { - std::construct_at(label_iter++, std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); - } - } - - auto [p, sample] = encoding::SampleCodec::decode(ptr, layout, default_timestamp_); - - ts.samples().emplace_back(sample); - } - - private: - std::string_view buffer_; - const BareBones::Vector& bytes_buffer_; - const MarkedMetric* item_{}; - Primitives::Timestamp default_timestamp_; - - PROMPP_ALWAYS_INLINE static const char* decode_count(const char* ptr, encoding::LayoutMarker& layout, uint32_t& labels_count) noexcept { - uint64_t chunk; - std::memcpy(&chunk, ptr, sizeof(chunk)); - - layout = encoding::LayoutMarker{.raw = static_cast(chunk)}; - - chunk >>= 8; - const uint64_t mask = (1ULL << BareBones::Bit::to_bits(layout.count_size_in_bytes())) - 1; - - labels_count = static_cast(chunk & mask); - - return ptr + 1 + layout.count_size_in_bytes(); - } - }; - - class Metadata { - public: - using MarkedT = MarkedMetadata; - - struct Context { - std::string_view buffer; - }; - - Metadata(const Context& ctx, const MarkedMetadata* item) : buffer_(ctx.buffer), item_(item) {} - - [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetadata* item() const noexcept { return item_; } - PROMPP_ALWAYS_INLINE void set_item(const MarkedMetadata* item) noexcept { item_ = item; } - - [[nodiscard]] PROMPP_ALWAYS_INLINE Prometheus::MetadataType type() const noexcept { return item_->type; } - [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view metric_name() const noexcept { return item_->metric_name.view(buffer_); } - [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view text() const noexcept { return item_->text.view(buffer_); } - - private: - std::string_view buffer_; - const MarkedMetadata* item_{}; - }; - - template - class MarkupBuffer { - public: - using MarkedT = typename T::MarkedT; - using Context = typename T::Context; - - class IteratorSentinel {}; - - class Iterator { - public: - using iterator_category = std::forward_iterator_tag; - using value_type = T; - using difference_type = ptrdiff_t; - using pointer = value_type*; - using reference = value_type&; - - Iterator(const Context& ctx, const MarkedT* ptr, uint32_t items_count) : item_(ctx, ptr), ptr_(ptr), items_count_(items_count), ctx_(ctx) {} - - [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type& operator*() const noexcept { return item_; } - [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type* operator->() const noexcept { return &item_; } - - PROMPP_ALWAYS_INLINE Iterator& operator++() noexcept { - item_.set_item(++ptr_); - --items_count_; - return *this; - } - - PROMPP_ALWAYS_INLINE Iterator operator++(int) noexcept { - auto tmp = *this; - ++(*this); - return tmp; - } - - PROMPP_ALWAYS_INLINE bool operator==(const IteratorSentinel&) const noexcept { return items_count_ == 0; } - - private: - T item_; - const MarkedT* ptr_; - uint32_t items_count_; - Context ctx_; - }; - - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t items_count() const noexcept { return buffer_.size(); } - [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return buffer_.allocated_memory(); } - - protected: - BareBones::Vector buffer_; - }; - - class MetricMarkupBuffer : public MarkupBuffer { - public: - using Base = MarkupBuffer; - using Iterator = typename Base::Iterator; - using IteratorSentinel = typename Base::IteratorSentinel; - - [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer, Primitives::Timestamp default_ts) const noexcept { - return {typename Base::Context{buffer, bytes_buffer_, default_ts}, this->buffer_.data(), this->items_count()}; - } - [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } - - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t bytes_count() const noexcept { return bytes_count_; } - void bytes_enlarge(uint32_t new_size) noexcept { - assert(new_size > bytes_count_); - if (new_size > bytes_buffer_.size()) [[likely]] { - bytes_buffer_.resize(new_size); - } - bytes_count_ = new_size; - } - - void bytes_shrink(uint32_t new_size) noexcept { bytes_count_ = new_size; } - - [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return this->buffer_.allocated_memory() + bytes_buffer_.allocated_memory(); } - - PROMPP_ALWAYS_INLINE void initialize(size_t reserve_bytes) noexcept { - this->buffer_.clear(); - - const size_t bytes_buffer_reserve = (reserve_bytes / 3) * 2; - const size_t items_buffer_reserve = (bytes_buffer_reserve / 3) / sizeof(MarkedMetric); - - this->buffer_.reserve(items_buffer_reserve); - bytes_buffer_.reserve(bytes_buffer_reserve); - bytes_count_ = 0; - } - - PROMPP_ALWAYS_INLINE void add_hash(uint64_t hash) noexcept { this->buffer_.back().hash = hash; } - - PROMPP_ALWAYS_INLINE void add_metric(uint32_t global_offset) noexcept { - this->buffer_.push_back(MarkedMetric{.hash = {}, .base_offset = global_offset, .data_offset = bytes_count()}); - } - - void add_layout(uint8_t layout) noexcept { - const uint32_t offset = bytes_count(); - *(bytes_buffer_.data() + offset) = layout; - bytes_shrink(offset + 1); - } - - void add_count(uint32_t count) noexcept { - const uint32_t bytes_written = (std::bit_width(count) + 7) / 8; - const uint32_t offset = bytes_count(); - std::memcpy(bytes_buffer_.data() + offset, &count, sizeof(count)); - bytes_shrink(offset + bytes_written); - } - - void add_label(MarkedLabel label) noexcept { - const auto end = - encoding::LabelCodec::encode(bytes_buffer_.data() + bytes_count(), label.name.offset, label.name.length, label.value.offset, label.value.length); - bytes_shrink(end - bytes_buffer_.data()); - } - - void add_sample(encoding::LayoutMarker layout, const Primitives::Sample& sample) noexcept { - using encoding::SampleValueType; - - char* data_ptr = bytes_buffer_.data(); - const char* end = encoding::SampleCodec::encode(data_ptr + bytes_count(), layout, sample); - - bytes_shrink(end - data_ptr); - } - - void add_padding() noexcept { - constexpr size_t kPaddingSizeBytes = 16; - bytes_enlarge(bytes_count() + kPaddingSizeBytes); - } - - private: - BareBones::Vector bytes_buffer_{}; - uint32_t bytes_count_ = 0; - }; - - class MetadataMarkupBuffer : public MarkupBuffer { - public: - using Base = MarkupBuffer; - using Iterator = typename Base::Iterator; - using IteratorSentinel = typename Base::IteratorSentinel; - - [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer) const noexcept { - return {typename Base::Context{buffer}, this->buffer_.data(), this->items_count()}; - } - [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } - - PROMPP_ALWAYS_INLINE void initialize(size_t reserve_bytes) noexcept { - this->buffer_.clear(); - const size_t items_buffer_reserve = reserve_bytes / sizeof(MarkedMetric); - this->buffer_.reserve(items_buffer_reserve); - } - - PROMPP_ALWAYS_INLINE void add(MarkedString metric_name, MarkedString text, Prometheus::MetadataType type) noexcept { - this->buffer_.emplace_back(metric_name, text, type); - } - }; - [[nodiscard]] Error parse_metadata() { static constexpr auto get_metadata_type = [](Token token) PROMPP_LAMBDA_INLINE { if (token == Token::kHelp) { From c54628e8ca7a437a06202b812ad67c1ea72f1259 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 10 Sep 2025 23:26:48 +0300 Subject: [PATCH 31/48] tidy fix --- pp/wal/hashdex/scraper/encoding_tests.cpp | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/pp/wal/hashdex/scraper/encoding_tests.cpp b/pp/wal/hashdex/scraper/encoding_tests.cpp index 0a33636232..e81c48590d 100644 --- a/pp/wal/hashdex/scraper/encoding_tests.cpp +++ b/pp/wal/hashdex/scraper/encoding_tests.cpp @@ -11,9 +11,12 @@ namespace { -using namespace PromPP::WAL::hashdex::scraper::encoding; using PromPP::Primitives::Sample; using PromPP::Primitives::Timestamp; +using PromPP::WAL::hashdex::scraper::encoding::LabelCodec; +using PromPP::WAL::hashdex::scraper::encoding::LayoutMarker; +using PromPP::WAL::hashdex::scraper::encoding::SampleCodec; +using PromPP::WAL::hashdex::scraper::encoding::SampleValueType; struct ValueTypeCase { double input; @@ -26,7 +29,7 @@ TEST_P(ValueTypeFixture, ClassifyValueCorrectly) { // Arrange // Act - const auto actual = value_type(GetParam().input); + const auto actual = PromPP::WAL::hashdex::scraper::encoding::value_type(GetParam().input); // Assert EXPECT_EQ(actual, GetParam().expected); @@ -60,12 +63,6 @@ INSTANTIATE_TEST_SUITE_P(ValueTypeTests, class SampleCodecFixture : public testing::Test { protected: static constexpr size_t kBufSize = 64; - std::pair&> encode_into_buffer(const LayoutMarker layout, Sample sample) { - buf_.fill(0); - char* start = buf_.data(); - char* end = SampleCodec::encode(start, layout, sample); - return {end, buf_}; - } SampleCodec::DecodeResult encode_and_decode(const LayoutMarker layout, Sample sample, int64_t default_ts = -1) { buf_.fill(0); From 2ac3cbf928ff256d69da39f264fa5f7f6ff72b1e Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 10 Sep 2025 23:49:37 +0300 Subject: [PATCH 32/48] x64-x86 double infinity fix --- pp/wal/hashdex/scraper/encoding.h | 2 +- pp/wal/hashdex/scraper/encoding_tests.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pp/wal/hashdex/scraper/encoding.h b/pp/wal/hashdex/scraper/encoding.h index 222d9b7aae..0d78a72b67 100644 --- a/pp/wal/hashdex/scraper/encoding.h +++ b/pp/wal/hashdex/scraper/encoding.h @@ -19,7 +19,7 @@ enum class SampleValueType : uint8_t { kUint32 = 0b0000'0000, kDouble, kUint8, k return SampleValueType::kZero; } - if (std::trunc(val) == val && val > 0.0) [[likely]] { + if (std::isfinite(val) && std::floor(val) == val && val > 0.0) [[likely]] { const auto uval = static_cast(val); if (uval <= std::numeric_limits::max()) { return SampleValueType::kUint8; diff --git a/pp/wal/hashdex/scraper/encoding_tests.cpp b/pp/wal/hashdex/scraper/encoding_tests.cpp index e81c48590d..ec857cc727 100644 --- a/pp/wal/hashdex/scraper/encoding_tests.cpp +++ b/pp/wal/hashdex/scraper/encoding_tests.cpp @@ -56,7 +56,7 @@ INSTANTIATE_TEST_SUITE_P(ValueTypeTests, ValueTypeCase{.input = 3.5, .expected = SampleValueType::kFloat}, ValueTypeCase{.input = -42.0, .expected = SampleValueType::kFloat}, - ValueTypeCase{.input = 1e300, .expected = SampleValueType::kDouble}, + ValueTypeCase{.input = 1e-10, .expected = SampleValueType::kDouble}, ValueTypeCase{.input = 3.141592653589793, .expected = SampleValueType::kDouble}, ValueTypeCase{.input = 0.1, .expected = SampleValueType::kDouble})); From 376aba8fdefcc89698e8ba9aa6e2dad891c18b5f Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Thu, 11 Sep 2025 12:40:43 +0300 Subject: [PATCH 33/48] small opts --- pp/wal/hashdex/scraper/encoding.h | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pp/wal/hashdex/scraper/encoding.h b/pp/wal/hashdex/scraper/encoding.h index 0d78a72b67..4c939d844e 100644 --- a/pp/wal/hashdex/scraper/encoding.h +++ b/pp/wal/hashdex/scraper/encoding.h @@ -19,7 +19,7 @@ enum class SampleValueType : uint8_t { kUint32 = 0b0000'0000, kDouble, kUint8, k return SampleValueType::kZero; } - if (std::isfinite(val) && std::floor(val) == val && val > 0.0) [[likely]] { + if (std::trunc(val) == val && val > 0.0 && val <= std::numeric_limits::max()) [[likely]] { const auto uval = static_cast(val); if (uval <= std::numeric_limits::max()) { return SampleValueType::kUint8; @@ -27,9 +27,7 @@ enum class SampleValueType : uint8_t { kUint32 = 0b0000'0000, kDouble, kUint8, k if (uval <= std::numeric_limits::max()) { return SampleValueType::kUint16; } - if (uval <= std::numeric_limits::max()) { - return SampleValueType::kUint32; - } + return SampleValueType::kUint32; } if (const auto f = static_cast(val); static_cast(f) == val) [[unlikely]] { @@ -242,16 +240,14 @@ class LabelCodec { if (v == 0) [[unlikely]] { return 0b00; } + std::memcpy(out, &v, sizeof(v)); if (v <= 0xFF) [[likely]] { - *out = static_cast(v); return 0b01; } if (v <= 0xFFFF) [[unlikely]] { - const auto value = static_cast(v); - std::memcpy(out, &value, sizeof(value)); return 0b10; } - std::memcpy(out, &v, sizeof(v)); + return 0b11; } From d87a4d68ae020401baff9a6c5a52997fdfe62297 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Thu, 11 Sep 2025 17:22:21 +0300 Subject: [PATCH 34/48] undo pop_back --- pp/bare_bones/vector.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pp/bare_bones/vector.h b/pp/bare_bones/vector.h index 221926f69a..cd58598b96 100644 --- a/pp/bare_bones/vector.h +++ b/pp/bare_bones/vector.h @@ -108,12 +108,6 @@ class GenericVector { derived()->set_size(0); } - PROMPP_ALWAYS_INLINE void pop_back() noexcept { - assert(!empty()); - - erase(end() - 1, end()); - } - PROMPP_ALWAYS_INLINE iterator erase(iterator first, iterator last) noexcept { assert(first >= begin()); assert(last <= end()); From 8c416ee37bc41421b4735f1706841180ea820a02 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Tue, 16 Sep 2025 13:20:32 +0300 Subject: [PATCH 35/48] review fixes --- pp/bare_bones/bit.h | 5 + pp/wal/hashdex/scraper/encoding.h | 235 ++++++++++++------ pp/wal/hashdex/scraper/encoding_tests.cpp | 290 +++++----------------- pp/wal/hashdex/scraper/marked.h | 77 ++---- pp/wal/hashdex/scraper/scraper.h | 32 +-- 5 files changed, 251 insertions(+), 388 deletions(-) diff --git a/pp/bare_bones/bit.h b/pp/bare_bones/bit.h index 98dd05ae46..5e8decabf2 100644 --- a/pp/bare_bones/bit.h +++ b/pp/bare_bones/bit.h @@ -78,6 +78,11 @@ constexpr Int to_ceil_units(Int bits) noexcept { constexpr uint8_t kUint64Bits = unit_bits; +template +constexpr T byte_width(T x) noexcept { + return to_ceil_bytes(std::bit_width(x)); +} + template PROMPP_ALWAYS_INLINE constexpr T be(T value) noexcept { if constexpr (std::endian::native == std::endian::little) { diff --git a/pp/wal/hashdex/scraper/encoding.h b/pp/wal/hashdex/scraper/encoding.h index 4c939d844e..8025b36821 100644 --- a/pp/wal/hashdex/scraper/encoding.h +++ b/pp/wal/hashdex/scraper/encoding.h @@ -2,7 +2,6 @@ #include #include -#include #include "bare_bones/bit.h" #include "primitives/sample.h" @@ -11,50 +10,27 @@ namespace PromPP::WAL::hashdex::scraper::encoding { enum class SampleValueType : uint8_t { kUint32 = 0b0000'0000, kDouble, kUint8, kUint16, kFloat, kZero, kNaN }; -[[nodiscard]] PROMPP_LAMBDA_INLINE inline SampleValueType value_type(const double val) noexcept { - if (std::isnan(val)) [[unlikely]] { - return SampleValueType::kNaN; - } - if (val == 0.0) [[unlikely]] { - return SampleValueType::kZero; - } - - if (std::trunc(val) == val && val > 0.0 && val <= std::numeric_limits::max()) [[likely]] { - const auto uval = static_cast(val); - if (uval <= std::numeric_limits::max()) { - return SampleValueType::kUint8; - } - if (uval <= std::numeric_limits::max()) { - return SampleValueType::kUint16; - } - return SampleValueType::kUint32; - } - - if (const auto f = static_cast(val); static_cast(f) == val) [[unlikely]] { - return SampleValueType::kFloat; - } - - return SampleValueType::kDouble; -} - struct LayoutMarker { - uint8_t raw; + bool has_ts : 1; + uint8_t length_bytes : 3; + SampleValueType sample_value_type : 4; - [[nodiscard]] PROMPP_LAMBDA_INLINE bool has_timestamp() const noexcept { return raw & 0b1000'0000; } + [[nodiscard]] PROMPP_LAMBDA_INLINE bool has_timestamp() const noexcept { return has_ts; } - [[nodiscard]] PROMPP_LAMBDA_INLINE uint8_t count_size_in_bytes() const noexcept { return (raw >> 4) & 0b0000'0111; } + [[nodiscard]] PROMPP_LAMBDA_INLINE uint8_t size_length_in_bytes() const noexcept { return length_bytes; } - [[nodiscard]] PROMPP_LAMBDA_INLINE SampleValueType value_type() const noexcept { return static_cast(raw & 0b0000'1111); } + [[nodiscard]] PROMPP_LAMBDA_INLINE SampleValueType value_type() const noexcept { return sample_value_type; } static PROMPP_LAMBDA_INLINE LayoutMarker make(bool has_ts, uint32_t labels_count, SampleValueType value_type) noexcept { - const uint8_t bytes_for_count = (std::bit_width(labels_count) + 7) / 8; - - return {static_cast((has_ts ? 0b1000'0000 : 0) | (bytes_for_count << 4) | (static_cast(value_type) & 0b0000'1111))}; + const uint8_t bytes_for_count = BareBones::Bit::to_ceil_bytes(std::bit_width(labels_count)); + return LayoutMarker{.has_ts = has_ts, .length_bytes = bytes_for_count, .sample_value_type = value_type}; } }; class SampleCodec { public: + static constexpr size_t kPreallocatedWriteSize = sizeof(Primitives::Sample); + static PROMPP_LAMBDA_INLINE char* encode(char* out, const LayoutMarker layout, Primitives::Sample sample) { using encoding::SampleValueType; @@ -125,6 +101,33 @@ class SampleCodec { return result; } + [[nodiscard]] static PROMPP_LAMBDA_INLINE SampleValueType value_type(const double val) noexcept { + if (std::isnan(val)) [[unlikely]] { + return SampleValueType::kNaN; + } + + if (val == 0.0) [[unlikely]] { + return SampleValueType::kZero; + } + + if (std::trunc(val) == val && val > 0.0 && val <= std::numeric_limits::max()) [[likely]] { + const auto uval = static_cast(val); + if (uval <= std::numeric_limits::max()) { + return SampleValueType::kUint8; + } + if (uval <= std::numeric_limits::max()) { + return SampleValueType::kUint16; + } + return SampleValueType::kUint32; + } + + if (const auto f = static_cast(val); static_cast(f) == val) [[unlikely]] { + return SampleValueType::kFloat; + } + + return SampleValueType::kDouble; + } + private: template PROMPP_ALWAYS_INLINE static char* write_value(char* out, const T& val) noexcept { @@ -135,6 +138,8 @@ class SampleCodec { class LabelCodec { public: + static constexpr size_t kPreallocatedWriteSize = sizeof(uint8_t) + 4 * sizeof(uint32_t); + static PROMPP_ALWAYS_INLINE char* encode(char* out, const uint32_t label_name_offset, const uint32_t label_name_length, @@ -142,6 +147,24 @@ class LabelCodec { const uint32_t label_value_length) noexcept { static constexpr uint8_t szm[4] = {0, 1, 2, 4}; + if (label_name_offset == 0 && label_name_length == 0) [[likely]] { + char* start = out++; + const uint8_t sz2 = push_and_encode(out, label_value_offset); + out += szm[sz2]; + const uint8_t sz3 = push_and_encode(out, label_value_length); + out += szm[sz3]; + *start = (sz2 << 4) | (sz3 << 6); + return out; + } + if ((label_name_offset | label_name_length | label_value_offset | label_value_length) <= 0xFF) [[likely]] { + const uint64_t chunk = static_cast(0b01010101) | static_cast(label_name_offset) << 8 | + static_cast(label_name_length) << 16 | static_cast(label_value_offset) << 24 | + static_cast(label_value_length) << 32; + + std::memcpy(out, &chunk, sizeof(chunk)); + return out + 5; + } + char* start = out++; const uint8_t sz0 = push_and_encode(out, label_name_offset); out += szm[sz0]; @@ -166,65 +189,78 @@ class LabelCodec { }; static PROMPP_ALWAYS_INLINE DecodeResult decode(const char* in) noexcept { - const char* start = in; - uint64_t chunk; std::memcpy(&chunk, in, sizeof(chunk)); const auto layout = static_cast(chunk); if (layout == 0b01010101) [[likely]] { - const auto name_off = static_cast(chunk >> 8); - const auto name_len = static_cast(chunk >> 16); - const auto value_off = static_cast(chunk >> 24); - const auto value_len = static_cast(chunk >> 32); + return decode_4_bytes(in, chunk); + } - return DecodeResult{ - .next = start + 5, .label_name_offset = name_off, .label_name_length = name_len, .label_value_offset = value_off, .label_value_length = value_len}; + if ((layout & 0x0F) == 0) [[likely]] { + return decode_value_only(in, chunk, layout); } - const uint8_t sz0 = layout & 0b11; - const uint8_t sz1 = (layout >> 2) & 0b11; + return decode_generic(++in, layout); + } + + private: + static PROMPP_ALWAYS_INLINE DecodeResult decode_4_bytes(const char* in, const uint64_t chunk) noexcept { + const auto name_off = static_cast(chunk >> 8); + const auto name_len = static_cast(chunk >> 16); + const auto value_off = static_cast(chunk >> 24); + const auto value_len = static_cast(chunk >> 32); + + return DecodeResult{ + .next = in + 5, .label_name_offset = name_off, .label_name_length = name_len, .label_value_offset = value_off, .label_value_length = value_len}; + } + + static PROMPP_ALWAYS_INLINE DecodeResult decode_value_only(const char* in, uint64_t chunk, const uint8_t layout) noexcept { const uint8_t sz2 = (layout >> 4) & 0b11; const uint8_t sz3 = (layout >> 6) & 0b11; - if ((layout & 0x0F) == 0) [[likely]] { - chunk >>= 8; - size_t used = 1; - uint32_t value_off = 0; - uint32_t value_len = 0; - if (sz2 == 0b01) [[likely]] { - value_off = static_cast(chunk); - used += sizeof(uint8_t); - chunk >>= BareBones::Bit::to_bits(sizeof(uint8_t)); - } else if (sz2 == 0b10) [[unlikely]] { - value_off = static_cast(chunk); - used += sizeof(uint16_t); - chunk >>= BareBones::Bit::to_bits(sizeof(uint16_t)); - } else if (sz2 == 0b11) [[unlikely]] { - value_off = static_cast(chunk); - used += sizeof(uint32_t); - chunk >>= BareBones::Bit::to_bits(sizeof(uint32_t)); - } - if (sz3 == 0b01) [[likely]] { - value_len = static_cast(chunk); - used += sizeof(uint8_t); - } else if (sz3 == 0b10) [[unlikely]] { - value_len = static_cast(chunk); - used += sizeof(uint16_t); - } else if (sz3 == 0b11) [[unlikely]] { - if (used + sizeof(uint32_t) <= sizeof(chunk)) [[likely]] { - value_len = static_cast(chunk); - } else [[unlikely]] { - std::memcpy(&value_len, start + used, sizeof(value_len)); - } - used += sizeof(value_len); + chunk >>= 8; + size_t used = 1; + + uint32_t value_off = 0; + uint32_t value_len = 0; + + if (sz2 == 0b01) [[likely]] { + value_off = static_cast(chunk); + used += sizeof(uint8_t); + chunk >>= BareBones::Bit::to_bits(sizeof(uint8_t)); + } else if (sz2 == 0b10) [[unlikely]] { + value_off = static_cast(chunk); + used += sizeof(uint16_t); + chunk >>= BareBones::Bit::to_bits(sizeof(uint16_t)); + } else if (sz2 == 0b11) [[unlikely]] { + value_off = static_cast(chunk); + used += sizeof(uint32_t); + chunk >>= BareBones::Bit::to_bits(sizeof(uint32_t)); + } + if (sz3 == 0b01) [[likely]] { + value_len = static_cast(chunk); + used += sizeof(uint8_t); + } else if (sz3 == 0b10) [[unlikely]] { + value_len = static_cast(chunk); + used += sizeof(uint16_t); + } else if (sz3 == 0b11) [[unlikely]] { + if (used + sizeof(uint32_t) <= sizeof(chunk)) [[likely]] { + value_len = static_cast(chunk); + } else [[unlikely]] { + std::memcpy(&value_len, in + used, sizeof(value_len)); } - - return DecodeResult{ - .next = start + used, .label_name_offset = 0, .label_name_length = 0, .label_value_offset = value_off, .label_value_length = value_len}; + used += sizeof(value_len); } - in += 1; + return DecodeResult{.next = in + used, .label_name_offset = 0, .label_name_length = 0, .label_value_offset = value_off, .label_value_length = value_len}; + } + + static PROMPP_ALWAYS_INLINE DecodeResult decode_generic(const char* in, const uint8_t layout) noexcept { + const uint8_t sz0 = layout & 0b11; + const uint8_t sz1 = (layout >> 2) & 0b11; + const uint8_t sz2 = (layout >> 4) & 0b11; + const uint8_t sz3 = (layout >> 6) & 0b11; const uint32_t name_off = read_val_partial(in, sz0); const uint32_t name_len = read_val_partial(in, sz1); @@ -235,7 +271,6 @@ class LabelCodec { .next = in, .label_name_offset = name_off, .label_name_length = name_len, .label_value_offset = value_off, .label_value_length = value_len}; } - private: static PROMPP_ALWAYS_INLINE uint8_t push_and_encode(char* out, uint32_t v) noexcept { if (v == 0) [[unlikely]] { return 0b00; @@ -271,4 +306,44 @@ class LabelCodec { } }; +class LayoutCountCodec { + public: + static constexpr size_t kPreallocatedWriteSize = sizeof(uint64_t); + + static PROMPP_ALWAYS_INLINE char* encode(char* out, const LayoutMarker layout, const uint32_t count) noexcept { + uint64_t chunk = 0; + std::memcpy(&chunk, &layout, sizeof(layout)); + chunk |= static_cast(count) << 8; + std::memcpy(out, &chunk, sizeof(chunk)); + + const uint32_t bytes_written = sizeof(layout) + layout.size_length_in_bytes(); + return out + bytes_written; + } + + struct DecodeResult { + const char* next; + LayoutMarker layout; + uint32_t count; + }; + + static PROMPP_ALWAYS_INLINE DecodeResult decode(const char* in) noexcept { + uint64_t chunk; + std::memcpy(&chunk, in, sizeof(chunk)); + + LayoutMarker layout{}; + std::memcpy(&layout, &chunk, sizeof(layout)); + + chunk >>= 8; + const uint64_t mask = (1ULL << BareBones::Bit::to_bits(layout.size_length_in_bytes())) - 1; + + auto labels_count = static_cast(chunk & mask); + + return {in + sizeof(layout) + layout.size_length_in_bytes(), layout, labels_count}; + } +}; + +static constexpr uint32_t metric_preallocated_bytes(uint32_t labels_count) noexcept { + return LayoutCountCodec::kPreallocatedWriteSize + LabelCodec::kPreallocatedWriteSize * labels_count + SampleCodec::kPreallocatedWriteSize; +} + } // namespace PromPP::WAL::hashdex::scraper::encoding \ No newline at end of file diff --git a/pp/wal/hashdex/scraper/encoding_tests.cpp b/pp/wal/hashdex/scraper/encoding_tests.cpp index ec857cc727..e1bbfbaa0f 100644 --- a/pp/wal/hashdex/scraper/encoding_tests.cpp +++ b/pp/wal/hashdex/scraper/encoding_tests.cpp @@ -29,7 +29,7 @@ TEST_P(ValueTypeFixture, ClassifyValueCorrectly) { // Arrange // Act - const auto actual = PromPP::WAL::hashdex::scraper::encoding::value_type(GetParam().input); + const auto actual = SampleCodec::value_type(GetParam().input); // Assert EXPECT_EQ(actual, GetParam().expected); @@ -60,7 +60,12 @@ INSTANTIATE_TEST_SUITE_P(ValueTypeTests, ValueTypeCase{.input = 3.141592653589793, .expected = SampleValueType::kDouble}, ValueTypeCase{.input = 0.1, .expected = SampleValueType::kDouble})); -class SampleCodecFixture : public testing::Test { +struct SampleCodecCase { + Sample sample; + bool has_ts; +}; + +class SampleCodecFixture : public testing::TestWithParam { protected: static constexpr size_t kBufSize = 64; @@ -79,155 +84,33 @@ class SampleCodecFixture : public testing::Test { const Timestamp default_ts_ = -1; }; -TEST_F(SampleCodecFixture, EncodeDecode_Uint32_WithTimestamp) { - // Arrange - Sample s{}; - s.value() = 123.0; - s.timestamp() = int64_t{42}; - - const auto layout = LayoutMarker::make(true, 0, SampleValueType::kUint32); - - // Act - auto res = encode_and_decode(layout, s, default_ts_); - - // Assert - EXPECT_DOUBLE_EQ(res.sample.value(), 123.0); - EXPECT_EQ(res.sample.timestamp(), 42); -} - -TEST_F(SampleCodecFixture, EncodeDecode_Double_WithoutTimestamp) { - // Arrange - Sample s{}; - s.value() = 3.141592653589793; - s.timestamp() = int64_t{12345}; - - const auto layout = LayoutMarker::make(false, 0, SampleValueType::kDouble); - - // Act - const auto res = encode_and_decode(layout, s, default_ts_); - - // Assert - EXPECT_DOUBLE_EQ(res.sample.value(), 3.141592653589793); - EXPECT_EQ(res.sample.timestamp(), default_ts_); -} - -TEST_F(SampleCodecFixture, EncodeDecode_Float_WithTimestamp) { - // Arrange - Sample s{}; - s.value() = 1.2345; - s.timestamp() = int64_t{111}; - - const auto layout = LayoutMarker::make(true, 0, SampleValueType::kFloat); - - // Act - const auto res = encode_and_decode(layout, s, default_ts_); - - // Assert - EXPECT_DOUBLE_EQ(res.sample.value(), static_cast(static_cast(1.2345))); - EXPECT_EQ(res.sample.timestamp(), 111); -} - -TEST_F(SampleCodecFixture, EncodeDecode_Uint8) { - // Arrange - Sample s{}; - s.value() = 255.0; - s.timestamp() = 7; - const auto layout = LayoutMarker::make(true, 0, SampleValueType::kUint8); - - // Act - const auto res = encode_and_decode(layout, s, default_ts_); - - // Assert - EXPECT_DOUBLE_EQ(res.sample.value(), 255.0); - EXPECT_EQ(res.sample.timestamp(), 7); -} - -TEST_F(SampleCodecFixture, EncodeDecode_Uint16) { - // Arrange - Sample s{}; - s.value() = 65535.0; - s.timestamp() = 9; - const auto layout = LayoutMarker::make(true, 0, SampleValueType::kUint16); - - // Act - const auto res = encode_and_decode(layout, s, default_ts_); - - // Assert - EXPECT_DOUBLE_EQ(res.sample.value(), 65535.0); - EXPECT_EQ(res.sample.timestamp(), 9); -} - -TEST_F(SampleCodecFixture, EncodeDecode_ZeroType_IgnoresOriginalValue) { - // Arrange - Sample s{}; - s.value() = 12345.0; - s.timestamp() = 222; - const auto layout = LayoutMarker::make(true, 0, SampleValueType::kZero); - - // Act - const auto res = encode_and_decode(layout, s, default_ts_); - - // Assert - EXPECT_DOUBLE_EQ(res.sample.value(), 0.0); - EXPECT_EQ(res.sample.timestamp(), 222); -} - -TEST_F(SampleCodecFixture, Decode_NaNType_ProducesNaN) { - // Arrange - Sample s{}; - s.value() = 777.0; - s.timestamp() = 333; - const auto layout = LayoutMarker::make(true, 0, SampleValueType::kNaN); - - // Act - const auto res = encode_and_decode(layout, s, default_ts_); - - // Assert - EXPECT_TRUE(std::isnan(res.sample.value())); - EXPECT_EQ(res.sample.timestamp(), 333); -} - -TEST_F(SampleCodecFixture, Decode_ReturnedPointerAdvancesCorrectly_NoTimestamp) { +TEST_P(SampleCodecFixture, CorrectSample) { // Arrange - Sample s{}; - s.value() = 42.0; - s.timestamp() = 1000; + const auto layout = LayoutMarker::make(GetParam().has_ts, 0, SampleCodec::value_type(GetParam().sample.value())); - const auto layout = LayoutMarker::make(false, 0, SampleValueType::kUint32); - - buf_.fill(0); - char* start = buf_.data(); - char* end = SampleCodec::encode(start, layout, s); + Sample expected = GetParam().sample; + if (!GetParam().has_ts) { + expected.timestamp() = default_ts_; + } // Act - const auto res = SampleCodec::decode(start, layout, default_ts_); + const auto res = encode_and_decode(layout, GetParam().sample, default_ts_); // Assert - EXPECT_EQ(res.next, end); - EXPECT_DOUBLE_EQ(res.sample.value(), 42.0); - EXPECT_EQ(res.sample.timestamp(), default_ts_); + EXPECT_EQ(res.sample, expected); } -TEST_F(SampleCodecFixture, Decode_ReturnedPointerAdvancesCorrectly_WithTimestamp) { - // Arrange - Sample s{}; - s.value() = 4242.0; - s.timestamp() = 2048; - - const auto layout = LayoutMarker::make(true, 0, SampleValueType::kDouble); - - buf_.fill(0); - char* start = buf_.data(); - char* end = SampleCodec::encode(start, layout, s); - - // Act - const auto res = SampleCodec::decode(start, layout, default_ts_); - - // Assert - EXPECT_EQ(res.next, end); - EXPECT_DOUBLE_EQ(res.sample.value(), 4242.0); - EXPECT_EQ(res.sample.timestamp(), 2048); -} +INSTANTIATE_TEST_SUITE_P(SampleCodecTests, + SampleCodecFixture, + testing::Values(SampleCodecCase{.sample = Sample(42, 123.0), .has_ts = true}, + SampleCodecCase{.sample = Sample(12345, std::numbers::pi), .has_ts = false}, + SampleCodecCase{.sample = Sample(111, 1.2345), .has_ts = true}, + SampleCodecCase{.sample = Sample(7, 255.0), .has_ts = true}, + SampleCodecCase{.sample = Sample(9, 65535.0), .has_ts = true}, + SampleCodecCase{.sample = Sample(222, 0.0), .has_ts = true}, + SampleCodecCase{.sample = Sample(333, PromPP::Prometheus::kNormalNan), .has_ts = true}, + SampleCodecCase{.sample = Sample(1000, 42.0), .has_ts = false}, + SampleCodecCase{.sample = Sample(2048, 4242.0), .has_ts = true})); class LabelCodecFixture : public testing::Test { protected: @@ -246,95 +129,6 @@ class LabelCodecFixture : public testing::Test { } }; -TEST_F(LabelCodecFixture, EncodeDecode_AllZeros) { - // Arrange - - // Act - const auto res = encode_and_decode(0, 0, 0, 0); - - // Assert - EXPECT_EQ(res.label_name_offset, 0u); - EXPECT_EQ(res.label_name_length, 0u); - EXPECT_EQ(res.label_value_offset, 0u); - EXPECT_EQ(res.label_value_length, 0u); -} - -TEST_F(LabelCodecFixture, EncodeDecode_OneByteValues) { - // Arrange - - // Act - const auto res = encode_and_decode(1, 2, 3, 4); - - EXPECT_EQ(res.label_name_offset, 1u); - EXPECT_EQ(res.label_name_length, 2u); - EXPECT_EQ(res.label_value_offset, 3u); - EXPECT_EQ(res.label_value_length, 4u); -} - -TEST_F(LabelCodecFixture, EncodeDecode_TwoByteValues) { - // Arrange - - // Act - const auto res = encode_and_decode(300, 400, 500, 600); - - EXPECT_EQ(res.label_name_offset, 300u); - EXPECT_EQ(res.label_name_length, 400u); - EXPECT_EQ(res.label_value_offset, 500u); - EXPECT_EQ(res.label_value_length, 600u); -} - -TEST_F(LabelCodecFixture, EncodeDecode_FourByteValues) { - // Arrange - - // Act - const auto res = encode_and_decode(100000, 200000, 300000, 400000); - - // Assert - EXPECT_EQ(res.label_name_offset, 100000u); - EXPECT_EQ(res.label_name_length, 200000u); - EXPECT_EQ(res.label_value_offset, 300000u); - EXPECT_EQ(res.label_value_length, 400000u); -} - -TEST_F(LabelCodecFixture, EncodeDecode_0044) { - // Arrange - - // Act - const auto res = encode_and_decode(0, 0, 300000, 400000); - - // Assert - EXPECT_EQ(res.label_name_offset, 0); - EXPECT_EQ(res.label_name_length, 0); - EXPECT_EQ(res.label_value_offset, 300000u); - EXPECT_EQ(res.label_value_length, 400000u); -} - -TEST_F(LabelCodecFixture, EncodeDecode_0024) { - // Arrange - - // Act - const auto res = encode_and_decode(0, 0, 1234, 123456); - - // Assert - EXPECT_EQ(res.label_name_offset, 0); - EXPECT_EQ(res.label_name_length, 0); - EXPECT_EQ(res.label_value_offset, 1234); - EXPECT_EQ(res.label_value_length, 123456); -} - -TEST_F(LabelCodecFixture, EncodeDecode_0124) { - // Arrange - - // Act - const auto res = encode_and_decode(0, 12, 1234, 123456); - - // Assert - EXPECT_EQ(res.label_name_offset, 0); - EXPECT_EQ(res.label_name_length, 12); - EXPECT_EQ(res.label_value_offset, 1234); - EXPECT_EQ(res.label_value_length, 123456); -} - TEST_F(LabelCodecFixture, FastPath_Layout01010101) { // Arrange buf_.fill(0); @@ -376,4 +170,36 @@ TEST_F(LabelCodecFixture, SimplifiedPath_NameFieldsZero) { EXPECT_EQ(res.next, buf_.data() + 1 + 1 + 2); } +struct LabelCase { + uint32_t name_off; + uint32_t name_len; + uint32_t value_off; + uint32_t value_len; +}; + +class LabelCodecParamFixture : public LabelCodecFixture, public testing::WithParamInterface {}; + +TEST_P(LabelCodecParamFixture, EncodeDecode) { + // Arrange + + // Act + const auto res = encode_and_decode(GetParam().name_off, GetParam().name_len, GetParam().value_off, GetParam().value_len); + + // Assert + EXPECT_EQ(res.label_name_offset, GetParam().name_off); + EXPECT_EQ(res.label_name_length, GetParam().name_len); + EXPECT_EQ(res.label_value_offset, GetParam().value_off); + EXPECT_EQ(res.label_value_length, GetParam().value_len); +} + +INSTANTIATE_TEST_SUITE_P(LabelCodecTests, + LabelCodecParamFixture, + testing::Values(LabelCase{.name_off = 0, .name_len = 0, .value_off = 0, .value_len = 0}, + LabelCase{.name_off = 1, .name_len = 2, .value_off = 3, .value_len = 4}, + LabelCase{.name_off = 300, .name_len = 400, .value_off = 500, .value_len = 600}, + LabelCase{.name_off = 100000, .name_len = 200000, .value_off = 300000, .value_len = 400000}, + LabelCase{.name_off = 0, .name_len = 0, .value_off = 300000, .value_len = 400000}, + LabelCase{.name_off = 0, .name_len = 0, .value_off = 1234, .value_len = 123456}, + LabelCase{.name_off = 0, .name_len = 12, .value_off = 1234, .value_len = 123456})); + } // namespace \ No newline at end of file diff --git a/pp/wal/hashdex/scraper/marked.h b/pp/wal/hashdex/scraper/marked.h index b1b8ad3c73..31e14f0b9a 100644 --- a/pp/wal/hashdex/scraper/marked.h +++ b/pp/wal/hashdex/scraper/marked.h @@ -1,9 +1,7 @@ #pragma once -#include #include -#include "bare_bones/bit.h" #include "bare_bones/vector.h" #include "encoding.h" #include "primitives/primitives.h" @@ -66,7 +64,7 @@ class Metric { struct Context { std::string_view buffer; - const BareBones::Vector& bytes_buffer; + const BareBones::Memory& bytes_buffer; Primitives::Timestamp default_timestamp{}; }; @@ -80,12 +78,11 @@ class Metric { template void read(Timeseries& ts) const { - const char* ptr = reinterpret_cast(bytes_buffer_.data() + item_->data_offset); + const char* ptr = bytes_buffer_.control_block().data + item_->data_offset; - uint32_t labels_count{}; - encoding::LayoutMarker layout{}; + const auto [next_ptr, layout, labels_count] = encoding::LayoutCountCodec::decode(ptr); + ptr = next_ptr; - ptr = decode_count(ptr, layout, labels_count); ts.label_set().resize(labels_count); auto label_iter = ts.label_set().begin(); @@ -108,23 +105,9 @@ class Metric { private: std::string_view buffer_; - const BareBones::Vector& bytes_buffer_; + const BareBones::Memory& bytes_buffer_; const MarkedMetric* item_{}; Primitives::Timestamp default_timestamp_; - - PROMPP_ALWAYS_INLINE static const char* decode_count(const char* ptr, encoding::LayoutMarker& layout, uint32_t& labels_count) noexcept { - uint64_t chunk; - std::memcpy(&chunk, ptr, sizeof(chunk)); - - layout = encoding::LayoutMarker{.raw = static_cast(chunk)}; - - chunk >>= 8; - const uint64_t mask = (1ULL << BareBones::Bit::to_bits(layout.count_size_in_bytes())) - 1; - - labels_count = static_cast(chunk & mask); - - return ptr + 1 + layout.count_size_in_bytes(); - } }; class Metadata { @@ -205,22 +188,22 @@ class MetricMarkupBuffer : public MarkupBuffer { using IteratorSentinel = typename Base::IteratorSentinel; [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer, Primitives::Timestamp default_ts) const noexcept { - return {typename Base::Context{buffer, bytes_buffer_, default_ts}, this->buffer_.data(), this->items_count()}; + return {typename Base::Context{buffer, bytes_buffer_, default_ts}, Base::buffer_.data(), Base::items_count()}; } [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t bytes_count() const noexcept { return bytes_count_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t bytes_count() const noexcept { return bytes_ptr_ - bytes_buffer_.control_block().data; } void bytes_enlarge(uint32_t new_size) noexcept { - assert(new_size > bytes_count_); - if (new_size > bytes_buffer_.size()) [[likely]] { - bytes_buffer_.resize(new_size); - } - bytes_count_ = new_size; - } + assert(new_size > bytes_count()); - void bytes_shrink(uint32_t new_size) noexcept { bytes_count_ = new_size; } + const uint32_t offset = bytes_count(); - [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return this->buffer_.allocated_memory() + bytes_buffer_.allocated_memory(); } + bytes_buffer_.grow_to_fit_at_least(new_size); + + bytes_ptr_ = bytes_buffer_.control_block().data + offset; + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return Base::buffer_.allocated_memory() + bytes_buffer_.allocated_memory(); } PROMPP_ALWAYS_INLINE void initialize(size_t reserve_bytes) noexcept { this->buffer_.clear(); @@ -229,8 +212,8 @@ class MetricMarkupBuffer : public MarkupBuffer { const size_t items_buffer_reserve = (bytes_buffer_reserve / 3) / sizeof(MarkedMetric); this->buffer_.reserve(items_buffer_reserve); - bytes_buffer_.reserve(bytes_buffer_reserve); - bytes_count_ = 0; + bytes_buffer_.resize_to_fit_at_least(bytes_buffer_reserve); + bytes_ptr_ = bytes_buffer_.control_block().data; } PROMPP_ALWAYS_INLINE void add_hash(uint64_t hash) noexcept { this->buffer_.back().hash = hash; } @@ -239,32 +222,18 @@ class MetricMarkupBuffer : public MarkupBuffer { this->buffer_.push_back(MarkedMetric{.hash = {}, .base_offset = global_offset, .data_offset = bytes_count()}); } - void add_layout(uint8_t layout) noexcept { - const uint32_t offset = bytes_count(); - *(bytes_buffer_.data() + offset) = layout; - bytes_shrink(offset + 1); - } - - void add_count(uint32_t count) noexcept { - const uint32_t bytes_written = (std::bit_width(count) + 7) / 8; - const uint32_t offset = bytes_count(); - std::memcpy(bytes_buffer_.data() + offset, &count, sizeof(count)); - bytes_shrink(offset + bytes_written); + void add_layout_and_count(const encoding::LayoutMarker layout, const uint32_t count) noexcept { + bytes_ptr_ = encoding::LayoutCountCodec::encode(bytes_ptr_, layout, count); } void add_label(MarkedLabel label) noexcept { - const auto end = - encoding::LabelCodec::encode(bytes_buffer_.data() + bytes_count(), label.name.offset, label.name.length, label.value.offset, label.value.length); - bytes_shrink(end - bytes_buffer_.data()); + bytes_ptr_ = encoding::LabelCodec::encode(bytes_ptr_, label.name.offset, label.name.length, label.value.offset, label.value.length); } void add_sample(encoding::LayoutMarker layout, const Primitives::Sample& sample) noexcept { using encoding::SampleValueType; - char* data_ptr = bytes_buffer_.data(); - const char* end = encoding::SampleCodec::encode(data_ptr + bytes_count(), layout, sample); - - bytes_shrink(end - data_ptr); + bytes_ptr_ = encoding::SampleCodec::encode(bytes_ptr_, layout, sample); } void add_padding() noexcept { @@ -273,8 +242,8 @@ class MetricMarkupBuffer : public MarkupBuffer { } private: - BareBones::Vector bytes_buffer_{}; - uint32_t bytes_count_ = 0; + BareBones::Memory bytes_buffer_; + char* bytes_ptr_{}; }; class MetadataMarkupBuffer : public MarkupBuffer { diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index c92eb0ff1b..cad69015b3 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -305,28 +305,24 @@ class Scraper { return parser_.parse_timestamp(marked_sample_.sample.timestamp(), marked_sample_.has_ts); } - void encode_metric_data(uint32_t metric_offset) noexcept { + void encode_metric_data(const uint32_t metric_offset) noexcept { metric_buffer_.add_metric(metric_offset); - const uint32_t bytes_offset = metric_buffer_.bytes_count(); - metric_buffer_.bytes_enlarge(bytes_offset + calculate_metric_prealloc_size(labels_.size())); - metric_buffer_.bytes_shrink(bytes_offset); + sort_and_filter_labels(); + append_labels_hash(); + + metric_buffer_.bytes_enlarge(metric_buffer_.bytes_count() + encoding::metric_preallocated_bytes(labels_.size())); const encoding::LayoutMarker layout = - encoding::LayoutMarker::make(marked_sample_.has_ts, labels_.size(), encoding::value_type(marked_sample_.sample.value())); - metric_buffer_.add_layout(layout.raw); + encoding::LayoutMarker::make(marked_sample_.has_ts, labels_.size(), encoding::SampleCodec::value_type(marked_sample_.sample.value())); + metric_buffer_.add_layout_and_count(layout, labels_.size()); - process_labels_buffer(metric_offset); + encode_labels(metric_offset); metric_buffer_.add_sample(layout, marked_sample_.sample); } - void process_labels_buffer(uint32_t offset) noexcept { - sort_and_filter_labels(); - append_labels_hash(); - - metric_buffer_.add_count(labels_.size()); - - for (auto& label : labels_) { + void encode_labels(const uint32_t offset) noexcept { + for (auto label : labels_) { if (!label.name.is_reserved_name()) [[likely]] { label.name.offset -= offset; } @@ -354,14 +350,6 @@ class Scraper { metric_buffer_.add_hash(hash.hash()); } - static PROMPP_ALWAYS_INLINE uint32_t calculate_metric_prealloc_size(uint32_t labels_count) noexcept { - constexpr uint32_t kCountVarintBytes = 8; - constexpr uint32_t kLabelBytes = 17; - constexpr uint32_t kSampleSize = 16; - - return kCountVarintBytes + labels_count * kLabelBytes + kSampleSize; - } - Parser parser_; MetricMarkupBuffer metric_buffer_; MetadataMarkupBuffer metadata_buffer_; From e2a0099ccee65b23a4005f173f98ce39c2a543d8 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Tue, 16 Sep 2025 15:36:03 +0300 Subject: [PATCH 36/48] more review fixes --- pp/wal/hashdex/scraper/encoding.h | 106 ++++++++++++++++++------------ pp/wal/hashdex/scraper/marked.h | 11 ++-- pp/wal/hashdex/scraper/scraper.h | 2 +- 3 files changed, 71 insertions(+), 48 deletions(-) diff --git a/pp/wal/hashdex/scraper/encoding.h b/pp/wal/hashdex/scraper/encoding.h index 8025b36821..d342f0e31b 100644 --- a/pp/wal/hashdex/scraper/encoding.h +++ b/pp/wal/hashdex/scraper/encoding.h @@ -29,9 +29,9 @@ struct LayoutMarker { class SampleCodec { public: - static constexpr size_t kPreallocatedWriteSize = sizeof(Primitives::Sample); + static constexpr size_t kMaximumEncodingSize = sizeof(Primitives::Sample); - static PROMPP_LAMBDA_INLINE char* encode(char* out, const LayoutMarker layout, Primitives::Sample sample) { + static char* encode(char* out, const LayoutMarker layout, Primitives::Sample sample) { using encoding::SampleValueType; const double val = sample.value(); @@ -59,7 +59,7 @@ class SampleCodec { Primitives::Sample sample; }; - static PROMPP_LAMBDA_INLINE DecodeResult decode(const char* in, const LayoutMarker layout, int64_t default_ts) { + static DecodeResult decode(const char* in, const LayoutMarker layout, int64_t default_ts) { using encoding::SampleValueType; DecodeResult result{}; @@ -101,7 +101,7 @@ class SampleCodec { return result; } - [[nodiscard]] static PROMPP_LAMBDA_INLINE SampleValueType value_type(const double val) noexcept { + [[nodiscard]] static SampleValueType value_type(const double val) noexcept { if (std::isnan(val)) [[unlikely]] { return SampleValueType::kNaN; } @@ -138,46 +138,22 @@ class SampleCodec { class LabelCodec { public: - static constexpr size_t kPreallocatedWriteSize = sizeof(uint8_t) + 4 * sizeof(uint32_t); - - static PROMPP_ALWAYS_INLINE char* encode(char* out, - const uint32_t label_name_offset, - const uint32_t label_name_length, - const uint32_t label_value_offset, - const uint32_t label_value_length) noexcept { - static constexpr uint8_t szm[4] = {0, 1, 2, 4}; + static constexpr size_t kMaximumEncodingSize = sizeof(uint8_t) + 4 * sizeof(uint32_t); + static char* encode(char* out, + const uint32_t label_name_offset, + const uint32_t label_name_length, + const uint32_t label_value_offset, + const uint32_t label_value_length) noexcept { if (label_name_offset == 0 && label_name_length == 0) [[likely]] { - char* start = out++; - const uint8_t sz2 = push_and_encode(out, label_value_offset); - out += szm[sz2]; - const uint8_t sz3 = push_and_encode(out, label_value_length); - out += szm[sz3]; - *start = (sz2 << 4) | (sz3 << 6); - return out; + return encode_value_only(out, label_value_offset, label_value_length); } - if ((label_name_offset | label_name_length | label_value_offset | label_value_length) <= 0xFF) [[likely]] { - const uint64_t chunk = static_cast(0b01010101) | static_cast(label_name_offset) << 8 | - static_cast(label_name_length) << 16 | static_cast(label_value_offset) << 24 | - static_cast(label_value_length) << 32; - std::memcpy(out, &chunk, sizeof(chunk)); - return out + 5; + if ((label_name_offset | label_name_length | label_value_offset | label_value_length) <= 0xFF) [[likely]] { + return encode_4_bytes(out, label_name_offset, label_name_length, label_value_offset, label_value_length); } - char* start = out++; - const uint8_t sz0 = push_and_encode(out, label_name_offset); - out += szm[sz0]; - const uint8_t sz1 = push_and_encode(out, label_name_length); - out += szm[sz1]; - const uint8_t sz2 = push_and_encode(out, label_value_offset); - out += szm[sz2]; - const uint8_t sz3 = push_and_encode(out, label_value_length); - out += szm[sz3]; - - *start = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); - - return out; + return encode_generic(out, label_name_offset, label_name_length, label_value_offset, label_value_length); } struct DecodeResult { @@ -188,7 +164,7 @@ class LabelCodec { uint32_t label_value_length; }; - static PROMPP_ALWAYS_INLINE DecodeResult decode(const char* in) noexcept { + static DecodeResult decode(const char* in) noexcept { uint64_t chunk; std::memcpy(&chunk, in, sizeof(chunk)); const auto layout = static_cast(chunk); @@ -205,6 +181,54 @@ class LabelCodec { } private: + static PROMPP_ALWAYS_INLINE char* encode_value_only(char* out, const uint32_t label_value_offset, const uint32_t label_value_length) noexcept { + static constexpr uint8_t szm[4] = {0, 1, 2, 4}; + char* start = out++; + + const uint8_t sz2 = push_and_encode(out, label_value_offset); + out += szm[sz2]; + const uint8_t sz3 = push_and_encode(out, label_value_length); + out += szm[sz3]; + + *start = (sz2 << 4) | (sz3 << 6); + + return out; + } + + static PROMPP_ALWAYS_INLINE char* encode_4_bytes(char* out, + const uint8_t label_name_offset, + const uint8_t label_name_length, + const uint8_t label_value_offset, + const uint8_t label_value_length) noexcept { + const uint64_t chunk = static_cast(0b01010101) | static_cast(label_name_offset) << 8 | static_cast(label_name_length) << 16 | + static_cast(label_value_offset) << 24 | static_cast(label_value_length) << 32; + + std::memcpy(out, &chunk, sizeof(chunk)); + return out + 5; + } + + static PROMPP_ALWAYS_INLINE char* encode_generic(char* out, + const uint32_t label_name_offset, + const uint32_t label_name_length, + const uint32_t label_value_offset, + const uint32_t label_value_length) noexcept { + static constexpr uint8_t szm[4] = {0, 1, 2, 4}; + + char* start = out++; + const uint8_t sz0 = push_and_encode(out, label_name_offset); + out += szm[sz0]; + const uint8_t sz1 = push_and_encode(out, label_name_length); + out += szm[sz1]; + const uint8_t sz2 = push_and_encode(out, label_value_offset); + out += szm[sz2]; + const uint8_t sz3 = push_and_encode(out, label_value_length); + out += szm[sz3]; + + *start = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); + + return out; + } + static PROMPP_ALWAYS_INLINE DecodeResult decode_4_bytes(const char* in, const uint64_t chunk) noexcept { const auto name_off = static_cast(chunk >> 8); const auto name_len = static_cast(chunk >> 16); @@ -308,7 +332,7 @@ class LabelCodec { class LayoutCountCodec { public: - static constexpr size_t kPreallocatedWriteSize = sizeof(uint64_t); + static constexpr size_t kMaximumEncodingSize = sizeof(uint64_t); static PROMPP_ALWAYS_INLINE char* encode(char* out, const LayoutMarker layout, const uint32_t count) noexcept { uint64_t chunk = 0; @@ -343,7 +367,7 @@ class LayoutCountCodec { }; static constexpr uint32_t metric_preallocated_bytes(uint32_t labels_count) noexcept { - return LayoutCountCodec::kPreallocatedWriteSize + LabelCodec::kPreallocatedWriteSize * labels_count + SampleCodec::kPreallocatedWriteSize; + return LayoutCountCodec::kMaximumEncodingSize + LabelCodec::kMaximumEncodingSize * labels_count + SampleCodec::kMaximumEncodingSize; } } // namespace PromPP::WAL::hashdex::scraper::encoding \ No newline at end of file diff --git a/pp/wal/hashdex/scraper/marked.h b/pp/wal/hashdex/scraper/marked.h index 31e14f0b9a..c37cd1df57 100644 --- a/pp/wal/hashdex/scraper/marked.h +++ b/pp/wal/hashdex/scraper/marked.h @@ -192,13 +192,10 @@ class MetricMarkupBuffer : public MarkupBuffer { } [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t bytes_count() const noexcept { return bytes_ptr_ - bytes_buffer_.control_block().data; } - void bytes_enlarge(uint32_t new_size) noexcept { - assert(new_size > bytes_count()); - + void bytes_enlarge(uint32_t extra_bytes) noexcept { const uint32_t offset = bytes_count(); - bytes_buffer_.grow_to_fit_at_least(new_size); + bytes_buffer_.grow_to_fit_at_least(offset + extra_bytes); bytes_ptr_ = bytes_buffer_.control_block().data + offset; } @@ -238,10 +235,12 @@ class MetricMarkupBuffer : public MarkupBuffer { void add_padding() noexcept { constexpr size_t kPaddingSizeBytes = 16; - bytes_enlarge(bytes_count() + kPaddingSizeBytes); + bytes_enlarge(kPaddingSizeBytes); } private: + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t bytes_count() const noexcept { return bytes_ptr_ - bytes_buffer_.control_block().data; } + BareBones::Memory bytes_buffer_; char* bytes_ptr_{}; }; diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index cad69015b3..6528bc85d5 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -311,7 +311,7 @@ class Scraper { sort_and_filter_labels(); append_labels_hash(); - metric_buffer_.bytes_enlarge(metric_buffer_.bytes_count() + encoding::metric_preallocated_bytes(labels_.size())); + metric_buffer_.bytes_enlarge(encoding::metric_preallocated_bytes(labels_.size())); const encoding::LayoutMarker layout = encoding::LayoutMarker::make(marked_sample_.has_ts, labels_.size(), encoding::SampleCodec::value_type(marked_sample_.sample.value())); From 371ae938a31e8c158c52822b7f4b7f7a65cd9c93 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 17 Sep 2025 10:15:02 +0300 Subject: [PATCH 37/48] more review fixes --- pp/wal/hashdex/scraper/encoding.h | 19 +++++++++---------- pp/wal/hashdex/scraper/scraper.h | 2 +- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/pp/wal/hashdex/scraper/encoding.h b/pp/wal/hashdex/scraper/encoding.h index d342f0e31b..392978919e 100644 --- a/pp/wal/hashdex/scraper/encoding.h +++ b/pp/wal/hashdex/scraper/encoding.h @@ -182,13 +182,12 @@ class LabelCodec { private: static PROMPP_ALWAYS_INLINE char* encode_value_only(char* out, const uint32_t label_value_offset, const uint32_t label_value_length) noexcept { - static constexpr uint8_t szm[4] = {0, 1, 2, 4}; char* start = out++; const uint8_t sz2 = push_and_encode(out, label_value_offset); - out += szm[sz2]; + out += szm_[sz2]; const uint8_t sz3 = push_and_encode(out, label_value_length); - out += szm[sz3]; + out += szm_[sz3]; *start = (sz2 << 4) | (sz3 << 6); @@ -212,17 +211,15 @@ class LabelCodec { const uint32_t label_name_length, const uint32_t label_value_offset, const uint32_t label_value_length) noexcept { - static constexpr uint8_t szm[4] = {0, 1, 2, 4}; - char* start = out++; const uint8_t sz0 = push_and_encode(out, label_name_offset); - out += szm[sz0]; + out += szm_[sz0]; const uint8_t sz1 = push_and_encode(out, label_name_length); - out += szm[sz1]; + out += szm_[sz1]; const uint8_t sz2 = push_and_encode(out, label_value_offset); - out += szm[sz2]; + out += szm_[sz2]; const uint8_t sz3 = push_and_encode(out, label_value_length); - out += szm[sz3]; + out += szm_[sz3]; *start = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); @@ -328,6 +325,8 @@ class LabelCodec { p += 4; return v; } + + static constexpr uint8_t szm_[4] = {0, 1, 2, 4}; }; class LayoutCountCodec { @@ -366,7 +365,7 @@ class LayoutCountCodec { } }; -static constexpr uint32_t metric_preallocated_bytes(uint32_t labels_count) noexcept { +static constexpr uint32_t metric_maximum_encoding_size(uint32_t labels_count) noexcept { return LayoutCountCodec::kMaximumEncodingSize + LabelCodec::kMaximumEncodingSize * labels_count + SampleCodec::kMaximumEncodingSize; } diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 6528bc85d5..6f9b2db5f1 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -311,7 +311,7 @@ class Scraper { sort_and_filter_labels(); append_labels_hash(); - metric_buffer_.bytes_enlarge(encoding::metric_preallocated_bytes(labels_.size())); + metric_buffer_.bytes_enlarge(encoding::metric_maximum_encoding_size(labels_.size())); const encoding::LayoutMarker layout = encoding::LayoutMarker::make(marked_sample_.has_ts, labels_.size(), encoding::SampleCodec::value_type(marked_sample_.sample.value())); From 212cedc75f6e19f517e4e9926b7f1afbfd70093a Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 17 Sep 2025 16:29:46 +0300 Subject: [PATCH 38/48] more review fixes --- pp/wal/hashdex/scraper/encoding.h | 30 ++++------- pp/wal/hashdex/scraper/encoding_tests.cpp | 26 ++++----- pp/wal/hashdex/scraper/marked.h | 64 ++--------------------- pp/wal/hashdex/scraper/marked_common.h | 59 +++++++++++++++++++++ 4 files changed, 88 insertions(+), 91 deletions(-) create mode 100644 pp/wal/hashdex/scraper/marked_common.h diff --git a/pp/wal/hashdex/scraper/encoding.h b/pp/wal/hashdex/scraper/encoding.h index 392978919e..8d525ab447 100644 --- a/pp/wal/hashdex/scraper/encoding.h +++ b/pp/wal/hashdex/scraper/encoding.h @@ -4,6 +4,7 @@ #include #include "bare_bones/bit.h" +#include "marked_common.h" #include "primitives/sample.h" #include "prometheus/value.h" @@ -140,28 +141,21 @@ class LabelCodec { public: static constexpr size_t kMaximumEncodingSize = sizeof(uint8_t) + 4 * sizeof(uint32_t); - static char* encode(char* out, - const uint32_t label_name_offset, - const uint32_t label_name_length, - const uint32_t label_value_offset, - const uint32_t label_value_length) noexcept { - if (label_name_offset == 0 && label_name_length == 0) [[likely]] { - return encode_value_only(out, label_value_offset, label_value_length); + static char* encode(char* out, const MarkedLabel label) noexcept { + if (label.name.offset == 0 && label.name.length == 0) [[likely]] { + return encode_value_only(out, label.value.offset, label.value.length); } - if ((label_name_offset | label_name_length | label_value_offset | label_value_length) <= 0xFF) [[likely]] { - return encode_4_bytes(out, label_name_offset, label_name_length, label_value_offset, label_value_length); + if ((label.name.offset | label.name.length | label.value.offset | label.value.length) <= 0xFF) [[likely]] { + return encode_4_bytes(out, label.name.offset, label.name.length, label.value.offset, label.value.length); } - return encode_generic(out, label_name_offset, label_name_length, label_value_offset, label_value_length); + return encode_generic(out, label.name.offset, label.name.length, label.value.offset, label.value.length); } struct DecodeResult { const char* next; - uint32_t label_name_offset; - uint32_t label_name_length; - uint32_t label_value_offset; - uint32_t label_value_length; + MarkedLabel label; }; static DecodeResult decode(const char* in) noexcept { @@ -232,8 +226,7 @@ class LabelCodec { const auto value_off = static_cast(chunk >> 24); const auto value_len = static_cast(chunk >> 32); - return DecodeResult{ - .next = in + 5, .label_name_offset = name_off, .label_name_length = name_len, .label_value_offset = value_off, .label_value_length = value_len}; + return DecodeResult{.next = in + 5, .label = {.name = {.offset = name_off, .length = name_len}, .value = {.offset = value_off, .length = value_len}}}; } static PROMPP_ALWAYS_INLINE DecodeResult decode_value_only(const char* in, uint64_t chunk, const uint8_t layout) noexcept { @@ -274,7 +267,7 @@ class LabelCodec { used += sizeof(value_len); } - return DecodeResult{.next = in + used, .label_name_offset = 0, .label_name_length = 0, .label_value_offset = value_off, .label_value_length = value_len}; + return DecodeResult{.next = in + used, .label = {.name = {.offset = 0, .length = 0}, .value = {.offset = value_off, .length = value_len}}}; } static PROMPP_ALWAYS_INLINE DecodeResult decode_generic(const char* in, const uint8_t layout) noexcept { @@ -288,8 +281,7 @@ class LabelCodec { const uint32_t value_off = read_val_partial(in, sz2); const uint32_t value_len = read_val_partial(in, sz3); - return DecodeResult{ - .next = in, .label_name_offset = name_off, .label_name_length = name_len, .label_value_offset = value_off, .label_value_length = value_len}; + return DecodeResult{.next = in, .label = {.name = {.offset = name_off, .length = name_len}, .value = {.offset = value_off, .length = value_len}}}; } static PROMPP_ALWAYS_INLINE uint8_t push_and_encode(char* out, uint32_t v) noexcept { diff --git a/pp/wal/hashdex/scraper/encoding_tests.cpp b/pp/wal/hashdex/scraper/encoding_tests.cpp index e1bbfbaa0f..941e403b21 100644 --- a/pp/wal/hashdex/scraper/encoding_tests.cpp +++ b/pp/wal/hashdex/scraper/encoding_tests.cpp @@ -120,7 +120,7 @@ class LabelCodecFixture : public testing::Test { LabelCodec::DecodeResult encode_and_decode(uint32_t name_off, uint32_t name_len, uint32_t value_off, uint32_t value_len) { buf_.fill(0); char* start = buf_.data(); - char* end = LabelCodec::encode(start, name_off, name_len, value_off, value_len); + char* end = LabelCodec::encode(start, PromPP::WAL::hashdex::scraper::MarkedLabel{name_off, name_len, value_off, value_len}); const auto res = LabelCodec::decode(start); @@ -143,10 +143,10 @@ TEST_F(LabelCodecFixture, FastPath_Layout01010101) { const auto res = LabelCodec::decode(buf_.data()); // Assert - EXPECT_EQ(res.label_name_offset, 10u); - EXPECT_EQ(res.label_name_length, 11u); - EXPECT_EQ(res.label_value_offset, 12u); - EXPECT_EQ(res.label_value_length, 13u); + EXPECT_EQ(res.label.name.offset, 10u); + EXPECT_EQ(res.label.name.length, 11u); + EXPECT_EQ(res.label.value.offset, 12u); + EXPECT_EQ(res.label.value.length, 13u); EXPECT_EQ(res.next, buf_.data() + 5); } @@ -163,10 +163,10 @@ TEST_F(LabelCodecFixture, SimplifiedPath_NameFieldsZero) { const auto res = LabelCodec::decode(buf_.data()); // Assert - EXPECT_EQ(res.label_name_offset, 0u); - EXPECT_EQ(res.label_name_length, 0u); - EXPECT_EQ(res.label_value_offset, 42u); - EXPECT_EQ(res.label_value_length, 1234u); + EXPECT_EQ(res.label.name.offset, 0u); + EXPECT_EQ(res.label.name.length, 0u); + EXPECT_EQ(res.label.value.offset, 42u); + EXPECT_EQ(res.label.value.length, 1234u); EXPECT_EQ(res.next, buf_.data() + 1 + 1 + 2); } @@ -186,10 +186,10 @@ TEST_P(LabelCodecParamFixture, EncodeDecode) { const auto res = encode_and_decode(GetParam().name_off, GetParam().name_len, GetParam().value_off, GetParam().value_len); // Assert - EXPECT_EQ(res.label_name_offset, GetParam().name_off); - EXPECT_EQ(res.label_name_length, GetParam().name_len); - EXPECT_EQ(res.label_value_offset, GetParam().value_off); - EXPECT_EQ(res.label_value_length, GetParam().value_len); + EXPECT_EQ(res.label.name.offset, GetParam().name_off); + EXPECT_EQ(res.label.name.length, GetParam().name_len); + EXPECT_EQ(res.label.value.offset, GetParam().value_off); + EXPECT_EQ(res.label.value.length, GetParam().value_len); } INSTANTIATE_TEST_SUITE_P(LabelCodecTests, diff --git a/pp/wal/hashdex/scraper/marked.h b/pp/wal/hashdex/scraper/marked.h index c37cd1df57..a0899601ec 100644 --- a/pp/wal/hashdex/scraper/marked.h +++ b/pp/wal/hashdex/scraper/marked.h @@ -4,60 +4,12 @@ #include "bare_bones/vector.h" #include "encoding.h" +#include "marked_common.h" #include "primitives/primitives.h" #include "prometheus/metric.h" -#include "prometheus/value.h" namespace PromPP::WAL::hashdex::scraper::inline marked { -#pragma pack(push, 1) -struct MarkedString { - uint32_t offset = 0; - uint32_t length = 0; - - [[nodiscard]] PROMPP_ALWAYS_INLINE static MarkedString create(std::string_view value, std::string_view buffer) noexcept { - return { - .offset = static_cast(value.data() - buffer.data()), - .length = static_cast(value.size()), - }; - } - - [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_reserved_name() const noexcept { return offset == 0 && length == 0; } - - [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_empty() const noexcept { return length == 0; } - - [[nodiscard]] std::string_view view(const std::string_view& buffer) const noexcept { - if (is_reserved_name()) [[unlikely]] { - return Prometheus::kMetricLabelName; - } - - return buffer.substr(offset, length); - } -}; - -struct MarkedLabel { - MarkedString name{}; - MarkedString value; -}; - -struct MarkedSample { - Primitives::Sample sample{}; - bool has_ts{}; -}; - -struct MarkedMetric { - uint64_t hash; - uint32_t base_offset; - uint32_t data_offset; -}; - -struct MarkedMetadata { - MarkedString metric_name{}; - MarkedString text{}; - Prometheus::MetadataType type{}; -}; -#pragma pack(pop) - class Metric { public: using MarkedT = MarkedMetric; @@ -86,16 +38,12 @@ class Metric { ts.label_set().resize(labels_count); auto label_iter = ts.label_set().begin(); - const char* base = buffer_.data() + item_->base_offset; for (uint32_t i = 0; i < labels_count; ++i) { - const auto [next_ptr, name_off, name_len, value_off, value_len] = encoding::LabelCodec::decode(ptr); + const auto [next_ptr, label] = encoding::LabelCodec::decode(ptr); ptr = next_ptr; - if (name_len == 0 && name_off == 0) [[unlikely]] { - std::construct_at(label_iter++, Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); - } else { - std::construct_at(label_iter++, std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); - } + const auto sub_buffer = buffer_.substr(item_->base_offset); + std::construct_at(label_iter++, label.name.view(sub_buffer), label.value.view(sub_buffer)); } auto [p, sample] = encoding::SampleCodec::decode(ptr, layout, default_timestamp_); @@ -223,9 +171,7 @@ class MetricMarkupBuffer : public MarkupBuffer { bytes_ptr_ = encoding::LayoutCountCodec::encode(bytes_ptr_, layout, count); } - void add_label(MarkedLabel label) noexcept { - bytes_ptr_ = encoding::LabelCodec::encode(bytes_ptr_, label.name.offset, label.name.length, label.value.offset, label.value.length); - } + void add_label(MarkedLabel label) noexcept { bytes_ptr_ = encoding::LabelCodec::encode(bytes_ptr_, label); } void add_sample(encoding::LayoutMarker layout, const Primitives::Sample& sample) noexcept { using encoding::SampleValueType; diff --git a/pp/wal/hashdex/scraper/marked_common.h b/pp/wal/hashdex/scraper/marked_common.h new file mode 100644 index 0000000000..8036173543 --- /dev/null +++ b/pp/wal/hashdex/scraper/marked_common.h @@ -0,0 +1,59 @@ +#pragma once + +#include + +#include "primitives/sample.h" +#include "prometheus/metric.h" +#include "prometheus/value.h" + +namespace PromPP::WAL::hashdex::scraper::inline marked { + +#pragma pack(push, 1) +struct MarkedString { + uint32_t offset = 0; + uint32_t length = 0; + + [[nodiscard]] PROMPP_ALWAYS_INLINE static MarkedString create(std::string_view value, std::string_view buffer) noexcept { + return { + .offset = static_cast(value.data() - buffer.data()), + .length = static_cast(value.size()), + }; + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_reserved_name() const noexcept { return offset == 0 && length == 0; } + + [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_empty() const noexcept { return length == 0; } + + [[nodiscard]] std::string_view view(const std::string_view& buffer) const noexcept { + if (is_reserved_name()) [[unlikely]] { + return Prometheus::kMetricLabelName; + } + + return buffer.substr(offset, length); + } +}; + +struct MarkedLabel { + MarkedString name{}; + MarkedString value; +}; + +struct MarkedSample { + Primitives::Sample sample{}; + bool has_ts{}; +}; + +struct MarkedMetric { + uint64_t hash; + uint32_t base_offset; + uint32_t data_offset; +}; + +struct MarkedMetadata { + MarkedString metric_name{}; + MarkedString text{}; + Prometheus::MetadataType type{}; +}; +#pragma pack(pop) + +} // namespace PromPP::WAL::hashdex::scraper::inline marked \ No newline at end of file From 97d1839d66100d2790fb02dace78beeadaa6d9a4 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 17 Sep 2025 18:25:16 +0300 Subject: [PATCH 39/48] tidy fix --- pp/wal/hashdex/scraper/encoding_tests.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pp/wal/hashdex/scraper/encoding_tests.cpp b/pp/wal/hashdex/scraper/encoding_tests.cpp index 941e403b21..4e65c3c406 100644 --- a/pp/wal/hashdex/scraper/encoding_tests.cpp +++ b/pp/wal/hashdex/scraper/encoding_tests.cpp @@ -120,7 +120,8 @@ class LabelCodecFixture : public testing::Test { LabelCodec::DecodeResult encode_and_decode(uint32_t name_off, uint32_t name_len, uint32_t value_off, uint32_t value_len) { buf_.fill(0); char* start = buf_.data(); - char* end = LabelCodec::encode(start, PromPP::WAL::hashdex::scraper::MarkedLabel{name_off, name_len, value_off, value_len}); + char* end = LabelCodec::encode(start, PromPP::WAL::hashdex::scraper::MarkedLabel{.name = {.offset = name_off, .length = name_len}, + .value = {.offset = value_off, .length = value_len}}); const auto res = LabelCodec::decode(start); From 65f4e2cf4d399f936ebbc1c0b1e93d541b0cee45 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 25 Mar 2026 14:32:36 +0000 Subject: [PATCH 40/48] bug fixes --- pp/wal/hashdex/scraper/marked.h | 2 +- pp/wal/hashdex/scraper/scraper.h | 41 +++++++++++++++----------------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/pp/wal/hashdex/scraper/marked.h b/pp/wal/hashdex/scraper/marked.h index 7409b2a5e9..b3c180388e 100644 --- a/pp/wal/hashdex/scraper/marked.h +++ b/pp/wal/hashdex/scraper/marked.h @@ -206,7 +206,7 @@ class MetadataMarkupBuffer : public MarkupBuffer { PROMPP_ALWAYS_INLINE void initialize(size_t reserve_bytes) noexcept { this->buffer_.clear(); - const size_t items_buffer_reserve = reserve_bytes / sizeof(MarkedMetric); + const size_t items_buffer_reserve = reserve_bytes / sizeof(typename Base::MarkedT); this->buffer_.reserve(items_buffer_reserve); } diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index b3538c59ca..ebd812470a 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -308,8 +308,8 @@ class Scraper { void encode_metric_data(const uint32_t metric_offset) noexcept { metric_buffer_.add_metric(metric_offset); - sort_and_filter_labels(); - append_labels_hash(); + const auto base_buffer = parser_.tokenizer().buffer(); + sort_and_filter_labels(base_buffer); metric_buffer_.bytes_enlarge(encoding::metric_maximum_encoding_size(labels_.size())); @@ -317,36 +317,33 @@ class Scraper { encoding::LayoutMarker::make(marked_sample_.has_ts, labels_.size(), encoding::SampleCodec::value_type(marked_sample_.sample.value())); metric_buffer_.add_layout_and_count(layout, labels_.size()); - encode_labels(metric_offset); + append_labels_hash_and_encode(metric_offset, base_buffer); metric_buffer_.add_sample(layout, marked_sample_.sample); } - void encode_labels(const uint32_t offset) noexcept { - for (auto label : labels_) { - if (!label.name.is_reserved_name()) [[likely]] { - label.name.offset -= offset; - } - label.value.offset -= offset; - - metric_buffer_.add_label(label); - } - } - - void sort_and_filter_labels() noexcept { + void sort_and_filter_labels(std::string_view base_buffer) noexcept { const auto it = std::remove_if(labels_.begin(), labels_.end(), [](const MarkedLabel& label) PROMPP_LAMBDA_INLINE { return label.value.is_empty(); }); labels_.erase(it, labels_.end()); - std::sort(labels_.begin(), labels_.end(), [buffer = parser_.tokenizer().buffer()](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { - return a.name.view(buffer) < b.name.view(buffer); - }); + const auto compare = [buffer = base_buffer](const MarkedLabel& a, const MarkedLabel& b) + PROMPP_LAMBDA_INLINE { return a.name.view(buffer) < b.name.view(buffer); }; + + std::sort(labels_.begin(), labels_.end(), compare); } - void append_labels_hash() noexcept { - const auto& tokenizer = parser_.tokenizer(); + void append_labels_hash_and_encode(uint32_t metric_offset, std::string_view base_buffer) noexcept { BareBones::XXHash3 hash; - for (const auto& label : labels_) { - hash.extend(label.name.view(tokenizer.buffer()), label.value.view(tokenizer.buffer())); + for (const auto& source_label : labels_) { + hash.extend(source_label.name.view(base_buffer), source_label.value.view(base_buffer)); + + MarkedLabel encoded_label = source_label; + if (!encoded_label.name.is_reserved_name()) [[likely]] { + encoded_label.name.offset -= metric_offset; + } + encoded_label.value.offset -= metric_offset; + metric_buffer_.add_label(encoded_label); } + metric_buffer_.add_hash(hash.hash()); } From adf420209b28f4598430772f1013743b09d6867a Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 29 Apr 2026 09:51:01 +0000 Subject: [PATCH 41/48] docs --- pp/2026-04-29-scraper-opt-article-draft.md | 559 +++++++++++++++++++++ pp/2026-04-29-scraper-opt-research.md | 413 +++++++++++++++ 2 files changed, 972 insertions(+) create mode 100644 pp/2026-04-29-scraper-opt-article-draft.md create mode 100644 pp/2026-04-29-scraper-opt-research.md diff --git a/pp/2026-04-29-scraper-opt-article-draft.md b/pp/2026-04-29-scraper-opt-article-draft.md new file mode 100644 index 0000000000..bd1a5577f8 --- /dev/null +++ b/pp/2026-04-29-scraper-opt-article-draft.md @@ -0,0 +1,559 @@ +# Замедлить нельзя ускорить: как мы сократили память Scraper в Prom++ в 3.33× и не потеряли в скорости + +> Уровень сложности: сложный +> Время на прочтение: ~ 15–20 мин + +> [РИСУНОК-ОБЛОЖКА] — иллюстрация-метафора: «упаковали чемодан плотнее, но успели на самолёт». Можно сделать схему «байт-в-байт» — слева жирный layout, справа компактный. + +Всем привет! Меня зовут Глеб Шигин, я C++-разработчик в команде Deckhouse Prom++. Это статья про то, как мы прошли путь от «сделали наивный POC, выиграли в памяти, но просели в скорости» до «выиграли и в памяти, и в чтении» — на одном hot-path куске кода и за неделю последовательных коммитов. + +Кратко предыстория. Prom++ — наш форк Prometheus, в котором ядро хранения и обработки горячих данных переписано на C++, при сохранении полной совместимости с Prometheus и его periphery на Go (см. предыдущие статьи: «Deckhouse Prom++: мы добавили плюсы к Prometheus и сократили потребление памяти в 7,8 раза», «FastCGo»). После ряда оптимизаций storage, индексов и WAL, очередным «жирным» местом по памяти оказался **Scraper** — компонент, который превращает «сырой» текстовый дамп `/metrics` от target-а в наш markup-буфер. + +Что будет в статье: + +- Разберём, что такое Scraper в Prom++ и как у него устроен markup. +- Посмотрим на распределение реальных данных и сделаем дубовый POC, который ужимает память в 3.65×, но ломает чтение. +- Превратим POC в рабочий формат с тестами, заплатив за это просадкой по скорости. +- Отыграем скорость обратно — с помощью unrolled-декодера, `over-allocate-then-shrink` и LZCNT. +- На каждом этапе будем мерить Google Benchmark + смотреть Tracy-трейс. + +Содержание: + +- Контекст и мотивация +- Исходное решение +- Гипотеза и POC +- Превращаем POC в рабочий формат +- Отыгрываем скорость: read/write opt +- Дочищаем sample и parse_metric +- Итог +- Выводы + +> [ЗАМЕР-СТЕНД] Все бенчмарки проводились на одной машине: укажи реальную модель CPU, OS, версию clang/gcc, флаг `-c opt --copt=-march=native`, отсутствие ASan/TSan, отсутствие частотных скачков. Это блок «как у Владимира в FastCGo-статье»: nice -n -20, фиксированный CPU governor performance, и т.п. + +## TL;DR + +- Profiled Scraper, увидели, что: + - на каждую метрику тратится **28 + 16·N** байт на markup (N — число labels) + ~50% overshoot за счёт `reserve(buffer.size()/2)`; + - реальные `name.length`, `value.length`, относительные offset-ы почти всегда умещаются в 1–2 байта; + - >95% значений — маленькие положительные целые counters, timestamp в большинстве случаев отсутствует. +- Сделали variable-length encoding для labels и samples + относительные offset'ы + отдельный header-массив `Vector` поверх variable-tail `Vector`. +- POC просел по скорости, но мы вернули её обратно за счёт unrolled-загрузок переменной длины, `resize+write+shrink` и `countl_zero` для расчёта длины. +- Итог: память **−70% (×3.33)**, read **+33..42%**, parse — небольшой проигрыш в худшем кейсе. + +> [ЗАМЕР-ИТОГ] График / таблица из 4 столбцов: parse, read, allocated_memory, ось X — ревизия. Две панели: cehp-like и m3-like дампы. Это ровно тот самый итоговый сводный график, на который дальше ссылаемся. + +## Контекст и мотивация + +Scraper — компонент, который раз в `scrape_interval` секунд получает HTTP-ответ от target-а в формате Prometheus text exposition (или OpenMetrics) и распарсивает его в нашу внутреннюю модель. Дальше его данные забирают шарды и складывают в head/WAL. Между «парсингом» и «забором шардом» данные живут в **markup-буфере** Scraper-а. + +``` +target /metrics ──HTTP──▶ PrometheusParser ──tokenize──▶ Scraper::parse ──▶ markup buffer + │ + ├── shard 0 ── metric.read(ts) + ├── shard 1 ── metric.read(ts) + └── ... +``` + +> [РИСУНОК-1] Аккуратная схема пайплайна. Хорошо бы показать, что markup-буфер живёт целиком, пока все шарды не вычитают свою половину/треть. + +Сколько мы сейчас платим за этот буфер? Берём типичный дамп `kube-apiserver /metrics` (~13 МБ текста, ~85 тысяч метрик) и пишем простой бенчмарк: + +```cpp +void ScraperParse(benchmark::State& state) { + ZoneScoped; + const auto str = get_file_content(); + std::string tmp_str; + tmp_str.resize(str.size()); + + for ([[maybe_unused]] auto _ : state) { + std::memcpy(tmp_str.data(), str.data(), str.size()); + PrometheusScraper scraper; + std::ignore = scraper.parse(tmp_str, 0); + } + + PrometheusScraper scraper; + auto tmp_str2 = str; + std::ignore = scraper.parse(tmp_str2, 0); + state.counters["Alloc"] = + benchmark::Counter(static_cast(scraper.allocated_memory()), + benchmark::Counter::kDefaults, + benchmark::Counter::OneK::kIs1024); +} +``` + +> [ЗАМЕР-BASELINE] Стартовые числа на baseline-коммите `3f0b2bd72`: +> +> - cehp-like дамп: ScraperParse ≈ 38 ms, ScraperRead ≈ 11 ms, Alloc ≈ **13.7 MiB**. +> - m3-like дамп: ScraperParse ≈ 23 ms, ScraperRead ≈ 1.8 ms, Alloc ≈ **13.7 MiB**. + +Чтобы было удобно, сразу зафиксируем три бенчмарка: + +- `Parser` — голая токенизация. Не KPI, нужна только чтобы вычитать «тело» из `ScraperParse`. +- `ScraperParse` — полный `Scraper::parse`. KPI по записи + `Alloc`. +- `ScraperRead` — полный проход `for (auto& m : metrics) m.read(ts)`. KPI по чтению. + +«Чистое время скрейпера» дальше будет фигурировать как `Plain Scraper Time = ScraperParse - Parser`. Это позволяет не путать прирост на токенизаторе с приростом на самом скрейпере. + +## Исходное решение + +Внутри Scraper лежит `MetricMarkupBuffer` — обычный `BareBones::Vector` (наша обёртка над контейнером с управляемой памятью). В него подряд складываются метрики: + +```cpp +#pragma pack(push, 1) +struct MarkedString { uint32_t offset; uint32_t length; }; // 8 B +struct MarkedLabel { MarkedString name; MarkedString value; }; // 16 B + +struct MarkedLabelSet { + uint32_t count; + MarkedLabel labels[]; +}; + +struct MarkedMetric { + uint64_t hash; // 8 + Primitives::Sample sample; // 16 (double value + int64 timestamp) + MarkedLabelSet label_set; // 4 + 16 * N +}; +#pragma pack(pop) +``` + +Итого на одну метрику: **28 + 16 · N** байт. Плюс при старте парсинга буфер резервируется как `metric_buffer_.initialize(buffer.size() / 2)`, что для 13 МБ дампа сразу даёт ~6.5 МБ и ещё 3.25 МБ на metadata. + +> [РИСУНОК-LAYOUT-V0] Картинка байт-в-байт: один `MarkedMetric` с тремя `MarkedLabel` подряд, цветом обозначить, что хранится. Подсветить, что на `__name__` тоже тратится 16 байт. + +На что мы посмотрели в первую очередь: + +> [ЗАМЕР-РАСПРЕДЕЛЕНИЕ] Гистограммы по реальному дампу: +> +> - длины `name`/`value` (доля укладывающихся в 1, 2, 3, 4 байта) — ожидаем, что 1–2 байта покрывают 95%+; +> - распределение типов значений: zero / uint8 / uint16 / uint32 / float / double / NaN — ожидаем доминирование small uint; +> - доля метрик с собственным timestamp — ожидаем, что подавляющее большинство без него. + +Эти три картинки и есть «топливо» для дальнейших гипотез. В нашей нагрузке: + +- 95+% `name.length` ≤ 255, 99+% `value.length` ≤ 65535; +- ~99% значений — целые положительные counters; +- timestamp в самой строке — единичные случаи (например, `pushgateway`). + +То есть мы платим за worst-case, которого почти никогда не бывает. + +## Гипотеза и POC + +«Если данных мало, давайте кодировать переменной длиной». Проблема в том, что переменная длина — это либо varint (с continue-битами), либо префиксы-длины. И то и другое — лишние ветки в hot loop. Поэтому первая попытка — два независимых POC: один — переменная упаковка labels, другой — переменная упаковка sample. Их **специально** делали без read-пути и без тестов, чтобы быстро увидеть «потолок» по памяти. + +### POC 1 — labels varint + +Идея простая. Каждый `MarkedString` хранит `offset` и `length`, оба — `uint32`. Вместо этого: + +- `offset`-ы делаем **относительными** к началу самой метрики. Тогда они почти всегда влезают в 1 байт. +- Каждое из 4 полей label-а (`name.offset`, `name.length`, `value.offset`, `value.length`) кодируем 1, 2, 3 или 4 байтами в зависимости от величины. +- В начале label-а кладём 1-байтовый **layout descriptor**, в котором по 2 бита на каждое из 4 полей описывают, сколько байт оно занимает. +- `__name__` — это специальный label, который встречается ровно один раз на метрику. Его имя не нужно хранить вовсе, обозначаем «(0, 0)» в полях имени и при чтении подставляем константу `Prometheus::kMetricLabelName`. +- Количество labels тоже varint: 1–5 байт. + +```cpp +const uint8_t sz0 = encode_size(label.name.offset); +const uint8_t sz1 = encode_size(label.name.length); +const uint8_t sz2 = encode_size(label.value.offset); +const uint8_t sz3 = encode_size(label.value.length); + +const uint8_t layout = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); + +std::array tmp{}; +char* out = tmp.data(); +*out++ = static_cast(layout); +*reinterpret_cast(out) = label.name.offset; out += sz0 + 1; +*reinterpret_cast(out) = label.name.length; out += sz1 + 1; +*reinterpret_cast(out) = label.value.offset; out += sz2 + 1; +*reinterpret_cast(out) = label.value.length; out += sz3 + 1; + +this->buffer_.push_back(tmp.data(), out); +``` + +Best case: **5 байт/label** против 16. Worst: 17. + +> [РИСУНОК-LABEL-LAYOUT] Картинка одного label-а: 1 байт layout + 4 переменных поля, цветом подсветить, какое поле сколько байт. На втором кадре — то же самое для `__name__` (ноль байт на name). + +Чтобы лейблы можно было упаковать в правильном порядке (с одинаковым хешем по сравнению с baseline), приходится сначала собрать их в скретч-вектор `BareBones::Vector labels_` (один на класс, `reserve(255)`), отсортировать по name, посчитать xxhash, и только потом батчем эмитить в основной буфер. Это +1 проход по labels на каждую метрику. + +### POC 2 — sample encoding + +Идея ещё проще. `Sample` — это всегда 16 байт (8 на double, 8 на timestamp). При том, что почти всегда: + +- timestamp равен default-у (т.е. его нет в дампе), +- value — небольшое положительное целое. + +Кодируем 1 байтом маркера + переменное тело: + +``` +bit 7 : has_ts (если 1 — за value идут 8 байт timestamp) +bits 0..3 : тип значения + 0000 zero (0 байт) + 0001 uint8 (1 байт) + 0010 uint16 (2 байта) + 0011 uint32 (4 байта) + 0100 staleNaN (0 байт) + 1000 float32 (4 байта) + 1001 double (8 байт) +``` + +Best case: **1 байт** на sample (нулевой counter без TS); worst — 17 байт (double + ts), почти не хуже фиксированных 16. + +```cpp +if (std::isnan(val)) [[unlikely]] { flush(0b00000100); return; } // NaN +if (val == 0.0) [[unlikely]] { flush(0b00000000); return; } // zero +if (std::trunc(val) == val && val > 0.0) [[likely]] { + auto ival = static_cast(val); + if (ival <= 0xFF) { flush(0b00000001); append((uint8_t)ival); return; } + if (ival <= 0xFFFF) { flush(0b00000010); append((uint16_t)ival); return; } + if (ival <= 0xFFFFFFFFULL) [[likely]] { flush(0b00000011); append((uint32_t)ival); return; } +} +float f = static_cast(val); +if (static_cast(f) == val) [[unlikely]] { flush(0b00001000); append(f); return; } // float32 +flush(0b00001001); append(val); // double +``` + +> [РИСУНОК-SAMPLE-LAYOUT] Маркер-байт + варианты тел. Хорошо смотрится «дерево» из 7 веток с подписанными байтами на каждом листе. + +### Что показал POC + +> [ЗАМЕР-POC] Прогон только `ScraperParse` + `Alloc` (read-пути ещё нет, поэтому соответствующие колонки пустые). Стиль — как у Владимира в FastCGo с benchstat-выводом. +> +> | Ревизия | Parse, ms (×) | Alloc, MiB (×) | Read | +> |---|---:|---:|---| +> | baseline `3f0b2bd72` | 38.2 (1.00) | 13.7 (1.00) | 11 ms | +> | + labels varint | 45.8 (0.83) | 5.0 (×2.74) | n/a | +> | + sample encoding | 47.8 (0.80) | 3.8 (×3.65) | n/a | +> | оба POC вместе (`4656f7137`) | 47.5 (0.80) | 4.15 (×3.31) | сломан | + +Итог POC: память жёстко сэкономили (×3.65), но: + +1. за это заплатили ~20% времени `parse` — добавились ветвистая запись label-а, цикл по 4 разрядностям, цикл по marker-типам; +2. сломали read-путь — он не реализован под новый формат, а старый perevich `Sample`/`MarkedLabel` уже не лежит в буфере; +3. тесты, естественно, красные. + +Это нормально и даже полезно: мы увидели **потолок** по памяти. Дальше задача — вернуть скорость и чтение, не отдав обратно эти 9.5 МБ. + +## Превращаем POC в рабочий формат + +Первое, что мы сделали — реализовали полный read-декодер и сделали тесты зелёными. Параллельно встретились с архитектурным неудобством: чтобы итерироваться по такому variable-length буферу, нужно либо знать длину каждого записанного элемента (т.е. ещё раз декодировать заголовок), либо хранить длину рядом. + +Мы выбрали третий путь — **разнесли header и tail по разным контейнерам**: + +```cpp +class MetricMarkupBuffer { + ... + BareBones::Vector metric_buffer_; // фиксированный header per metric (16 B) + BareBones::Vector bytes_buffer_; // variable tail: count + labels + sample +}; + +struct MarkedMetric { + uint64_t hash; // 8 + uint32_t base_offset; // 4 — начало метрики в исходном буфере (для относительных offset-ов) + uint32_t data_offset; // 4 — начало tail в bytes_buffer_ +}; +``` + +> [РИСУНОК-LAYOUT-V1] Двухэтажная схема: сверху `Vector` — все «шапки» подряд, снизу `Vector` — все упакованные «хвосты» подряд. Стрелочка `data_offset` от шапки к хвосту. + +Что это даёт: + +- `++iterator` теперь — это `++ptr_` по плотному `MarkedMetric*`. Не нужно декодировать tail только чтобы посчитать длину. +- На `metric_buffer_` (16 B/запись) шарды ходят кеш-friendly при чисто «hash-обходах». +- `Sample` целиком переехал в variable tail; `default_timestamp` поднялся на уровень всего `Scraper`-а — один на дамп. Это ещё минус 8 байт «обычной» метрики. + +Read-декодер на эту схему получается прямолинейно: + +```cpp +template +void read(Timeseries& ts) const { + const char* ptr = bytes_buffer_.data() + item_->data_offset; + const char* base = buffer_.data() + item_->base_offset; + + uint32_t labels_count = decode_varint(ptr); + ts.label_set().reserve(labels_count); + + for (uint32_t i = 0; i < labels_count; ++i) { + const uint8_t layout = static_cast(*ptr++); + const uint8_t sz0 = (layout >> 0) & 0x3; + const uint8_t sz1 = (layout >> 2) & 0x3; + const uint8_t sz2 = (layout >> 4) & 0x3; + const uint8_t sz3 = (layout >> 6) & 0x3; + + auto read_val = [&](uint8_t sz) PROMPP_LAMBDA_INLINE -> uint32_t { + uint32_t v = 0; + memcpy(&v, ptr, sz + 1); // <-- внимание на эту строку + ptr += sz + 1; + return v; + }; + + uint32_t name_off = read_val(sz0); + uint32_t name_len = read_val(sz1); + uint32_t value_off = read_val(sz2); + uint32_t value_len = read_val(sz3); + + // ... append ... + } + // ... sample ... +} +``` + +Тесты зелёные. Сериализация и десериализация совместимы. Smoke-pass идёт. + +> [ЗАМЕР-READ-WORKS] Та же таблица, но добавились колонки `Pass` и `Read`: +> +> | Ревизия | Parse (×) | Pass (×) | Read (×) | Alloc (×) | +> |---|---:|---:|---:|---:| +> | baseline | 1.00 | 1.00 | 1.00 | 1.00 | +> | `4656f7137` initial encoding | 0.80 | n/a | broken | 3.31 | +> | `0027219f0` read works | 0.76 | 0.70 | **0.65** | 3.33 | + +Read стал **медленнее** baseline. Не на единицы процентов — на 35%. + +> «Просчитался, но где?!» + +Заглядываем в Tracy (включаем `--//bazel/toolchain:profiling=1`, в `Metric::read` и `parse_metric` выставляем `ZoneScopedN` на под-фазы — `decode_count`, `decode_labels`, `decode_sample`): + +> [РИСУНОК-TRACY-READ-V1] Скриншот Tracy для `Metric::read` на `0027219f0`. Видно, что `decode_labels` занимает абсолютное большинство времени, а внутри него — узкое место в строке `memcpy(&v, ptr, sz + 1)`. + +Дело в той самой строчке `memcpy(&v, ptr, sz + 1)` с **переменным** размером. Компилятор не разворачивает её в простой load — он эмитит `memcpy`-fallback (или generic loop). Каждый label-байт стоит отдельной инструкции call/branch. Аналогично `add_label` на write-пути сначала пишет в `std::array tmp`, потом делает `push_back(tmp, len)` — лишний копирующий проход. + +## Отыгрываем скорость: read/write opt + +Это самый важный коммит во всей серии — `9b62e6ae`. Здесь мы возвращаем скорость, не теряя ни байта памяти. + +### Read: разворачиваем «memcpy с переменной длиной» руками + +```cpp +PROMPP_ALWAYS_INLINE +static uint32_t read_val_partial(const char*& p, uint8_t sz) noexcept { + if (sz == 0) [[likely]] { + const uint32_t v = static_cast(p[0]); + p += 1; + return v; + } + if (sz == 1) { + const uint32_t v = static_cast(static_cast(p[0])) + | (static_cast(static_cast(p[1])) << 8); + p += 2; + return v; + } + if (sz == 2) { + const uint32_t v = static_cast(static_cast(p[0])) + | (static_cast(static_cast(p[1])) << 8) + | (static_cast(static_cast(p[2])) << 16); + p += 3; + return v; + } + uint32_t v; + std::memcpy(&v, p, 4); + p += 4; + return v; +} +``` + +После этого декодирование одного label-а становится цепочкой прямых нагружений + сдвигов. Switch по типу sample заодно отсортировали по частоте: `uint8 / zero / uint16 / uint32 / staleNaN / float / double`. `decode_varint` и `read_val_partial` пометили `PROMPP_ALWAYS_INLINE` — без этого компилятор для коротких функций делал отдельные `call`. + +### Write: `over-allocate-then-shrink` + +Та же идея, но в зеркале. Раньше `add_label` писал в локальный `std::array tmp` и делал `push_back(tmp, used_size)`. Это два прохода. + +Стало: + +```cpp +const uint32_t bytes_needed = sizeof(layout) + (sz0 + sz1 + sz2 + sz3) + 4; +const uint32_t offset = bytes_count(); +bytes_buffer_.resize(bytes_buffer_.size() + 17); // worst-case заранее +char* out = bytes_buffer_.data() + offset; + +*out++ = static_cast(layout); +std::memcpy(out, &label.name.offset, sz0 + 1); out += sz0 + 1; +std::memcpy(out, &label.name.length, sz1 + 1); out += sz1 + 1; +std::memcpy(out, &label.value.offset, sz2 + 1); out += sz2 + 1; +std::memcpy(out, &label.value.length, sz3 + 1); + +bytes_buffer_.resize(offset + bytes_needed); // shrink to actual +``` + +Что мы здесь выиграли: + +- В `bytes_buffer_` пишем **сразу**, без промежуточного буфера. +- `resize(+17)` обеспечивает «есть куда писать»; внутри `memcpy(out, &val, sz+1)` нет проверок границ. +- В конце `resize` сжимает до фактически записанного размера. У `BareBones::Vector` это просто корректировка `size_`, без пересборки. + +Тот же приём применили к `add_count` (varint длиной до 5 байт). + +### Write: `encode_size` через LZCNT + +Раньше: + +```cpp +static uint8_t encode_size(uint32_t v) noexcept { + if (v <= 0xFF) return 0; + if (v <= 0xFFFF) return 1; + if (v <= 0xFFFFFF) return 2; + return 3; +} +``` + +Стало: + +```cpp +PROMPP_ALWAYS_INLINE static uint8_t encode_size(uint32_t v) noexcept { + const uint32_t msb = (v == 0 ? 0 : 31 - std::countl_zero(v)); + return msb >> 3; +} +``` + +Каскад из 3 веток заменили на одну инструкцию `lzcnt`/`bsr` + сдвиг. На hot-path этот вызов идёт ровно 4 раза на каждый label, поэтому экономия чувствуется. + +### Что это дало + +> [ЗАМЕР-RW-OPT] Прогон 3 бенчмарков: +> +> | Ревизия | Parse (×) | Pass (×) | Read (×) | Alloc (×) | +> |---|---:|---:|---:|---:| +> | `0027219f0` read works | 0.76 | 0.70 | 0.65 | 3.33 | +> | `9b62e6ae` r/w opt | **0.83** | **0.79** | **1.39** (cehp) / **1.39** (m3) | 3.33 | + +Read **+114%** на cehp-like дампе и **+289%** на m3-like (за счёт более коротких labels). Pass прибавил ~12%. Память — ровно на месте. + +> [РИСУНОК-TRACY-READ-V2] Тот же скриншот Tracy `Metric::read`, но после `9b62e6ae`. `decode_labels` ужалось примерно в 2 раза. + +## Дочищаем sample и parse_metric + +После главного «win-back» осталось два аккуратных шага. + +### `aca48508` — тот же приём для samples + +`add_sample` страдал от той же болезни, что и `add_label` до `9b62e6ae`: внутренний `flush()` лямбдой, отдельные `push_back` для маркера, для значения, для timestamp. Применяем ровно то же — `resize(+17)`, последовательный курсор, в конце `resize(actual)`: + +```cpp +constexpr uint32_t max_sample_bytes = 1 + sizeof(sample.sample); // 1 marker + 8 value + 8 ts +bytes_buffer_.resize(offset + max_sample_bytes); +char* out = bytes_buffer_.data() + offset; +char* start = out; + +if (std::isnan(val)) [[unlikely]] { + *out++ = static_cast((has_ts ? 0b10000000 : 0) | 0b00000100); +} else if (val == 0.0) [[unlikely]] { + *out++ = static_cast((has_ts ? 0b10000000 : 0) | 0b00000000); +} else if (std::trunc(val) == val && val > 0.0) [[likely]] { + // ... uint8 / uint16 / uint32 / fallback double ... +} else { + // ... float32 / double ... +} + +if (has_ts) { + std::memcpy(out, &sample.sample.timestamp(), 8); + out += 8; +} + +bytes_buffer_.resize(offset + (out - start)); +``` + +Параллельно поменяли layout: timestamp **после** value (раньше — до). На декодер влияет минимально, но позволяет в записи использовать один общий курсор без переустановок. + +> [ЗАМЕР-SAMPLE-OPT] Очень небольшая прибавка в pass (cehp 34.3 → 33.6 ms, m3 16.9 → 16.3 ms), read — почти шум. Но это «бесплатное» улучшение в стиле уже сделанного. + +### `2036d7ea` — `MetricParser` → метод + +В старой схеме на каждую метрику стек-локально создавался объект: + +```cpp +if (const auto error = MetricParser{parser_, metric_buffer_, labels_, + static_cast(tokenizer.token_str().data() - tokenizer.buffer().data()), + default_timestamp}.parse(); + error != Error::kNoError) [[unlikely]] { + metric_buffer_.remove_item(); + return error; +} +``` + +`MetricParser` — это вложенный класс с 4 ссылками-членами. Компилятор имеет полное право его не инлайнить — каждая метрика стоит лишнего конструктора + indirection через ref-поля. + +Превратили в обычный метод `Scraper`: + +```cpp +[[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_metric() { + ... +} +``` + +Все «ссылки» теперь — обычные `this->labels_`, `this->metric_buffer_`. `PROMPP_ALWAYS_INLINE` теперь работает по-настоящему. + +> [ЗАМЕР-INLINE] Pass cehp 34.0 → 33.2 ms, m3 17.4 → 16.3 ms. Прибавка маленькая, но стабильная и видна на каждом прогоне. На Tracy исчез отдельный фрейм конструктора `MetricParser`. + +> [РИСУНОК-TRACY-INLINE] Сравнение двух Tracy-кадров: до и после, видно отсутствие лишнего фрейма. + +Дальше (`1113d04b`) — чистая косметика: вынес `process_labels_buffer`, `sort_and_filter_labels`, `append_labels_hash` в отдельные методы. На скорости/памяти не сказывается, но код становится читаемым. + +## Итог + +> [ТАБЛИЦА-ИТОГ] Сводная таблица из исследования. Структура: ревизия → ключевое изменение → Parse, Pass, Read, Alloc. Можно вставить из `2026-04-29-scraper-opt-research.md` секция 7 — её достаточно «причесать». + +Чтобы было нагляднее, итог одной картинкой: + +> [РИСУНОК-ИТОГ] 4-панельный график (или 2×2): по оси X — ревизии в хронологическом порядке, по Y — нормированное к baseline время/память. Панели: +> +> 1. Plain Scraper Time (cehp) +> 2. Plain Scraper Time (m3) +> 3. Read time (cehp/m3 на одной картинке) +> 4. Allocated memory +> +> Пунктирной линией — baseline = 1.0. Видны три «горки»: подъём `4656f7137` / `0027219f0`, спуск `9b62e6ae`, плоский конец. + +Финальный layout одной метрики: + +> [РИСУНОК-LAYOUT-FINAL] Сверху — массив `MarkedMetric` (16 B на запись). Снизу — variable tail в `bytes_buffer_`: varint count → последовательность labels (1 байт layout + 4..16 байт полей) → 1 байт marker + value + опц. timestamp. + +Финальные числа: + +- На наших дампах **markup-память ужалась в 3.33×** (с 13.7 МБ до 4.13 МБ). +- На «толстом» дампе read стал на **33% быстрее**, на плотном — на **42% быстрее**. +- На write — небольшой проигрыш (от −5% до −20% относительно baseline в зависимости от дампа). В нашем pipeline write делается один раз на интервал scrape-а, read — N раз на N шардов, поэтому сделка для нас — отличная. + +## Выводы + +- **Domain-specific knowledge — главный инструмент**. Все приёмы в этой статье — `varint`, layout-byte, `__name__`-elision, `default_timestamp` per-scraper — обоснованы тем, **как реально выглядят** Prometheus-дампы. Без гистограмм по реальным данным мы бы не выбрали 4-разрядное layout-кодирование (а взяли бы скучный `LEB128`/`varint`, и проиграли бы по скорости из-за continue-битов). +- **`memcpy` с переменной длиной — не SIMD**. Если длина — известный compile-time-набор из 1..4 байт, разверните руками. Это вернуло нам ~2× на read. +- **Over-allocate-then-shrink**. Если максимальная длина записи известна и невелика, проще сделать `resize(+max)`, писать линейно без проверок границ, а в конце `resize(actual)`. У нас это вторая половина выигрыша. +- **Считайте байты вычислениями**. `(31 - countl_zero(v)) >> 3` короче и быстрее каскада `if`, и читается ничем не хуже. +- **Inline-friendly код**. Вложенные классы с ref-членами на hot-path — ловушка. Если функция вызывается на каждой метрике — лучше метод того же класса с `ALWAYS_INLINE`. +- **Профилируйте на каждом шаге**. Tracy + Google Benchmark показывают разницу между «кажется, должно быть быстрее» и «вот тут стоит ровно тот memcpy». В нашем случае POC, который выглядел как «победа», на read-пути оказался регрессом — и без трейсов мы бы это пропустили. + +> [ЗАМЕР-FINAL-SUMMARY] Здесь хорошо смотрится один итоговый бар-чарт: «было — стало» по 4 метрикам, желательно на двух типах нагрузки. + +## Что осталось за кадром + +В этой статье мы не разбираем: + +- что происходит с `metadata_buffer_` (`# HELP / # TYPE / # UNIT`) — там оптимизаций пока не делали; +- как именно `BareBones::Vector` обрабатывает `resize(+N)` без полной реаллокации; +- интеграцию Scraper-а с шардами (это отдельная история про `hash() % N`-распределение). + +Если интересно про что-то из этого — пишите в комментариях. + +## P. S. + +Читайте также в нашем блоге: + +- Deckhouse Prom++: мы добавили плюсы к Prometheus и сократили потребление памяти в 7,8 раза +- FastCGo: как мы ускорили вызов C-кода в Go в 16,5 раза +- (другие наши статьи по Prom++) + +Теги: `prom++`, `prometheus`, `c++`, `optimization`, `bit packing`, `varint`, `tracy`, `google benchmark` + +--- + +## Чек-лист правок и допросов перед публикацией + +> Это служебный блок для нас, в финальный текст не идёт. + +- [ ] Проверить, что числа в TL;DR сходятся с финальным графиком. +- [ ] Подтвердить термины: «scraper» / «скрейпер» — выбрать одно написание и придерживаться. +- [ ] Все имена коммитов оставить как есть (в стиле автора-оригинала) либо переименовать в человекопонятные (например, `4656f7137` → «POC: initial encoding»). +- [ ] Решить, показываем ли мы фактические Tracy-скриншоты или схематично рисуем «как было / как стало». +- [ ] Снять все три гистограммы распределения с реальных дампов. +- [ ] Перепрогнать все замеры на одной чистой машине, единым прогоном через `bench_sweep.sh`. +- [ ] Свести итоговый график. +- [ ] Указать ссылку на репозиторий с примерами / на сам Prom++. diff --git a/pp/2026-04-29-scraper-opt-research.md b/pp/2026-04-29-scraper-opt-research.md new file mode 100644 index 0000000000..826353c1eb --- /dev/null +++ b/pp/2026-04-29-scraper-opt-research.md @@ -0,0 +1,413 @@ +# Scraper optimization — research dump + +Сводка по серии коммитов на ветке `scraper-rw-opt`, в которой `Scraper` +в `pp/wal/hashdex/scraper/scraper.h` был оптимизирован по памяти и +скорости. Используется как рабочий black-book перед написанием статьи +на Хабр. + +Период: `3f0b2bd72` (2025-08-25) → `1113d04b` (2025-09-01). + +## 1. Что измеряется + +Бенчмарки лежат в `pp/wal/benchmarks/scraper_benchmark.cpp`. + +- `BenchmarkParser` — голая токенизация (`PrometheusParser::tokenizer().tokenize` + `next()` в цикле). Это **не KPI**, а вычитаемая база, чтобы получить «чистое» время скрейпера: `Plain Scraper Time = ScraperParse - Parser`. +- `BenchmarkScraperParse` — полный `Scraper::parse(buffer, 0)` + замер `allocated_memory()`. +- `BenchmarkScraperRead` — после одного `parse()` итерируем по `scraper.metrics()` и каждому `metric.read(ts)` заполняем `TimeseriesSemiview`. + +Две колонки в твоей CSV: + +- `cehp-runner` — более «толстый» промовский дамп (длинные labels, смешанные значения; `parse ≈ 38..50 ms`, alloc baseline ≈ 13.7 MiB). +- `m3` — более плотный дамп (короткие labels, в основном целые counters; `parse ≈ 23..30 ms`, alloc такой же). + +Файлы фикстур у тебя локальные. В `scraper.flags` указан путь +`…/wal/benchmarks/data/blobs.txt`, в репо его нет. Сейчас в `/prompp/pp/` +лежит только один дамп — `kube-api.metrics` (~13M, 85k строк, стиль +ближе к cehp-runner). + +## 2. Базовый layout (`3f0b2bd72`) + +```cpp +struct MarkedString { uint32 offset; uint32 length; }; // 8 B +struct MarkedLabel { MarkedString name; MarkedString value; }; // 16 B +struct MarkedLabelSet { uint32 count; MarkedLabel labels[]; }; +struct MarkedMetric { + uint64 hash; // 8 + Sample sample; // 16 (double + int64) + MarkedLabelSet ls; // 4 + 16*N +}; +``` + +Итого на одну метрику: **28 + 16·N байт** + `metric_buffer_.initialize(buffer.size()/2)` overshoot. + +Поверх — `MetricParser{...}.parse()` инстанцируется на каждую метрику, +читает токены, по ходу вызывает `markup_buffer_.add_label(...)`, в +конце — `calculate_hash()` (sort + xxhash). + +Сам коммит `3f0b2bd72` — это разделение одного `BenchmarkScraper` на +`Parse`/`Read` + добавление счётчика `Alloc`. Это и есть baseline. + +«Жирное» в этом виде: +1. На `__name__` тратится полный `MarkedLabel` 16 B — у каждой метрики. +2. Большинство `offset`/`length` в реальной нагрузке умещаются в 1–2 байта, а резервируются 4. +3. `Sample` всегда 16 B, хотя ~99% значений в Prom-text — маленькие положительные целые counters/gauges. +4. `metric_buffer_.initialize(buffer.size()/2)` берёт «с запасом», что и даёт 13.7 MiB. + +## 3. Хронология коммитов + +| # | Hash | Тема | +|---|------|------| +| 0 | `3f0b2bd72` | benchmark update (baseline) | +| POC₁ | (не закоммичено) | **labels varint** — POC переменной упаковки label-полей | +| POC₂ | (не закоммичено) | **sample encoding** — POC переменной упаковки value/timestamp | +| 1 | `4656f7137` | initial encoding — оба POC сведены воедино, тесты ещё не проходят, read «болт» | +| 2 | `0027219f0` | read + passing all tests — реализован декодер, разбили хранилище на `metric_buffer_` (header) + `bytes_buffer_` (variable tail) | +| 3 | `9b62e6ae` | read/write optimization — главный прирост по скорости | +| 4 | `aca48508` | sample optimization — тот же приём для samples | +| 5 | `26d0af47` | benchmark only: read лишь чётных hash (имитация sharding) | +| 6 | `2036d7ea` | `MetricParser` → метод `parse_metric()` (inline-friendly) | +| 7 | `1113d04b` | extract-method косметика над `parse_metric` | + +POC-шаги `labels varint` и `sample encoding` в git-истории отдельно +не сохранились — их собрали в коммит `4656f7137` («initial encoding»). +Сами замеры по ним есть в CSV: только `parse, ns` и `alloc, Mi`, +без `pass` и без `read`. Это согласуется с тем, что: + +- read-декодера на тот момент ещё не было (в `MarkedMetric` стоял placeholder `uint8_t bytes[]` без реального сериализатора); +- `Plain Scraper Time` зависит от рабочего read-пути, который тоже ещё не считался. + +То есть POC-числа — это ровно те моменты, когда уже сжимали запись и +хотели увидеть, **на сколько падает alloc** и **сколько за это платим +в parse**, не задумываясь про корректность чтения. + +## 4. Что и зачем в каждом коммите + +### 4.1. POC₁ «labels varint» (часть `4656f7137`) + +**Гипотеза.** На реальном Prom-тексте `offset`/`length` для name и value +почти всегда умещаются в 1 байт; offset метрики тоже невелик, если +хранить его относительно начала самой метрики; `__name__` встречается +ровно один раз на метрику и его имя не нужно хранить вовсе. + +**Что сделано.** +- Дополнительное поле в `MarkedMetric`: `offset` — глобальное смещение «первого» токена метрики в исходном буфере. Все остальные `MarkedString::offset` хранятся относительно него. +- Каждая `MarkedLabel` сериализуется как: + - 1 байт **layout descriptor**: `(sz0)|(sz1<<2)|(sz2<<4)|(sz3<<6)`, где каждое `szk ∈ [0..3]` означает «занимает `szk+1` байтов». + - далее 4 поля `name.offset`, `name.length`, `value.offset`, `value.length`, по 1..4 байта каждое. + - best case: **5 байт/label** (vs 16); worst case: 17. +- `__name__` детектируется через `is_reserved_name()` → name.offset/length = 0/0 (декодер потом восстановит). +- Количество label'ов — **varint 1..5 байт** (вместо 4 фиксированных). +- Лейблы сначала собираются в скретч-вектор `labels_` (`reserve(255)` на класс), сортируются по name, по ним считается xxhash, и только потом батчем пишутся в основной буфер. +- Снят `metric_buffer_.initialize(buffer.size()/2)` overshoot. Буфер растёт по ходу. + +**Цена.** Ветвистая по 4 разрядностям запись + копирование labels через скретч + сортировка. По CSV: parse 38.2 → 45.8 ms (cehp), 22.8 → 28.8 ms (m3). Зато **alloc 13.7 → 5.0 MiB (×2.74)**. + +### 4.2. POC₂ «sample encoding» (часть `4656f7137`) + +**Гипотеза.** `Sample` 16 B всегда — расточительно: TS обычно +отсутствует (используется default), value — почти всегда маленький +положительный uint, реже — float, изредка — NaN/staleNaN. + +**Что сделано.** Один маркер-байт: + +``` +bit 7 : has_ts (если 1 — за value идут 8 байт timestamp) +bits 0..3 : тип значения + 0000 zero (0 байт) + 0001 uint8 (1 байт) + 0010 uint16 (2 байта) + 0011 uint32 (4 байта) + 0100 staleNaN (0 байт) + 1000 float32 (4 байта) + 1001 double (8 байт) +``` + +Best case: **1 байт** на sample (нулевой counter без TS); worst: +17 байт (double + ts) — почти не хуже фиксированных 16. + +Ветвление при записи: `isnan` → `==0.0` → `trunc(val)==val && val>0` +(uint-фастпас) → `float32-fits-double` → `double`. + +**Числа.** Alloc 5.02 → 3.77 MiB (×3.65). Скорость parse чуть просела: +45.8 → 47.8 ms. Видимо именно из-за добавленных ветвлений в hot loop. + +### 4.3. `4656f7137` «initial encoding» — оба POC + первая нормализация + +Свели обе схемы вместе и привели код к виду, в котором уже можно +писать read-декодер. Read-декодера в нём ещё нет (`MarkedMetric::occupied_size()` +оставлен с FIXME-комментом и `bytes[]` placeholder), а `read()` ничего +разумного не делает. Поэтому числа Read/Pass для него у тебя «N/A» +в CSV (только parse и alloc). + +Замеры на cehp: parse 47.5 ms, alloc 4.15 MiB (×3.31). Это и есть +«дешёвая фаза» — мы максимально экономим память, не заботясь о +скорости чтения. + +### 4.4. `0027219f0` «read + passing all tests» + +**Главное архитектурное решение.** `MetricMarkupBuffer` перестал быть +одним сплошным `Vector`. Стало два контейнера: + +```cpp +BareBones::Vector metric_buffer_; // фиксированный header per metric +BareBones::Vector bytes_buffer_; // variable tail (varint count + labels + sample) +``` + +`MarkedMetric { uint64 hash; uint32 base_offset; uint32 data_offset; }` — фиксированные **16 байт**. + +**Зачем это.** +- `++Iterator` теперь это `++ptr_` по массиву `MarkedMetric*`. До этого нужно было читать `occupied_size()`, который для упакованного формата требовал бы декодировать tail только чтобы посчитать длину. +- На `metric_buffer_` можно сделать `std::sort` без перетаскивания variable tail. +- Hash, base/data offsets лежат плотно — кеш-friendly при чисто «по hash» проходах. + +**Кроме этого.** +- Реализован полный `Metric::read(ts)`: декодирует varint count → распаковывает labels по layout-байту → разбирает sample по маркеру. Под `__name__` подставляет `Prometheus::kMetricLabelName`. +- `default_timestamp` поднят на уровень `Scraper` (один на дамп) и подставляется в `read()`, если у sample не было `has_ts`. Это минус 8 байт на «обычную» метрику. + +**Числа.** Read-декодер «как написан» — медленнее baseline (16.9 ms vs 10.9 на cehp), потому что цикл декодирования label'а делает `memcpy(&v, ptr, sz+1)` с **переменной длиной**, и компилятор не может развернуть это в простую загрузку. Pass тоже просел (38.7 vs 27.0). Alloc — практически тот же 4.13 MiB. Это «честный» промежуточный шаг: формат заработал end-to-end, но дорого по времени. Тесты идут зелёные. + +### 4.5. `9b62e6ae` «read/write optimization» — самый важный коммит по скорости + +**Read.** +- `read_val_partial(ptr, sz)` развёрнут на 4 ветки: sz=0 → один байт, sz=1 → `b0|b1<<8`, sz=2 → `b0|b1<<8|b2<<16`, sz=3 → `memcpy 4 байт`. Это убирает `memcpy` с переменным размером. +- `decode_varint` и `read_val_partial` — `PROMPP_ALWAYS_INLINE`. +- Switch по типу sample отсортирован по частоте (`uint8/zero/uint16/uint32/staleNaN/float/double`). +- Для uint16 — ручная сборка из двух байт со сдвигом (а не memcpy). + +**Write.** Та же идея, что в read, только наоборот: +- `add_count`: вместо `push_back(tmp_array, len)` стало `bytes_buffer_.resize(size + N); *out = …` — пишем сразу в буфер, без промежуточного массива. Размер `N` известен в каждой ветке. +- `add_label`: `resize(size + 17)` (worst case заранее), пишем 4 поля по `memcpy(out, &val, sz+1)`, в конце `resize(offset + bytes_needed)` — «over-allocate then shrink». Это позволяет компилятору эмитить безусловные `mov` без проверок границ внутри. +- `encode_size(v)` — заменён на `(31 - countl_zero(v)) >> 3` (LZCNT/BSR), вместо каскада `if`-ов. + +**Числа.** Cehp: read **16.9 → 7.9 ms (×2.14)**, pass 38.7 → 34.3, parse 50.0 → 46.1. M3: read **4.98 → 1.29 ms (×3.86)**. + +### 4.6. `aca48508` «sample optimization» + +Применили тот же приём `resize+write+shrink` к `add_sample`: + +```cpp +constexpr uint32_t max_sample_bytes = 1 + sizeof(Sample); // 17 +bytes_buffer_.resize(offset + max_sample_bytes); +char* out = ...; *out++ = marker; std::memcpy(out, &val, sizeof(T)); +if (has_ts) { memcpy(out, &ts, 8); out += 8; } +bytes_buffer_.resize(offset + (out - start)); +``` + +Плюс: timestamp кодируется **после** value (раньше — до). На декодер +влияет минимально, но позволяет использовать один общий `out` курсор +без условных переустановок. + +**Числа.** Cehp: pass 34.3 → 33.6, read небольшой шум. M3: pass 16.9 → 16.3. + +### 4.7. `26d0af47` «scraper read benchmark 2 shards» + +Только бенчмарк: `if (metric.hash() % 2 == 0) metric.read(ts);`. Это +меняет **базовую цифру** read и pass — здесь у тебя в CSV вторая +«серия» с новым 3f0b2bd72-baseline (33 ms на pass cehp, 11.3 ms m3). +Все три числа после этого замерены в этой новой системе координат. + +### 4.8. `2036d7ea` «MetricParser → parse_metric method» + +Ранее на каждую метрику конструировался +`MetricParser{parser_, markup_buffer_, labels_, global_offset, default_timestamp}.parse()`. +У вложенного класса с ссылками на 4 объекта компилятор не всегда может +полноценно заинлайнить `parse()`. + +После рефакторинга `parse_metric()` — обычный метод `Scraper`, помеченный +`PROMPP_ALWAYS_INLINE`. Все ссылки — это просто члены того же класса. + +**Числа.** Pass cehp 34.0 → 33.2, m3 17.4 → 16.3. Виден реальный, хоть и небольшой, прирост. + +### 4.9. `1113d04b` «parse_metric refactoring» + +Чистая косметика: вынес `process_labels_buffer`, `sort_and_filter_labels`, +`append_labels_hash` из тела `parse_metric` в отдельные методы. Поведение +и численно — то же. + +## 5. Что зашло, а что нет + +| Коммит | Идея | Память | Скорость parse | Скорость read | Стоит включать в статью | +|---|---|---|---|---|---| +| 4656f7137 (POC labels varint) | переменная упаковка 4 полей label'а + offset relative + дроп `__name__` | ✅ ×2.74 | ⛔ просел | n/a (нет read) | **Да, ключевой шаг** | +| 4656f7137 (POC sample) | 1-байт маркер + 0..8 байт value + опц. ts | ✅ ×3.65 | ⛔ ещё просел | n/a | **Да** | +| 4656f7137 (initial encoding) | свод POC + relative-offset через `base_offset` + дроп overshoot reserve | ✅ ×3.31 (стабильное) | ⛔ медленный pass | ⛔ read «битый» | **Да, как «end of POC»** | +| 0027219f0 | split на `metric_buffer_`+`bytes_buffer_` + рабочий read | = | ⛔ дальше просел | ⛔ медленный | **Да, объясняет архитектуру и трейдоф** | +| 9b62e6ae | read_val_partial unroll, encode_size через LZCNT, resize+write+shrink на write | = | ✅ -10..15% | ✅ ×2.1 (cehp), ×3.9 (m3) | **Да, главный «win-back» по скорости** | +| aca48508 | тот же приём для samples | = | ≈ | ≈ | средне, упомянуть как «продолжение приёма» | +| 26d0af47 | bench 2 shards | n/a | n/a | n/a | методологический момент в статье | +| 2036d7ea | inline-friendly parse_metric | = | ✅ -2..5% | ≈ | **Да, как «чистый» рефактор с замером** | +| 1113d04b | косметика | = | ≈ | ≈ | можно опустить | + +**Итог по серии 1 vs baseline (cehp):** alloc ×3.33, read ×1.33, plain pass ×0.81 — то есть по памяти выиграли ~3.3×, по времени read получили буст ~1.3×, но потеряли ~20% на pass. + +**Итог по серии 1 vs baseline (m3):** alloc ×3.33, read ×1.42, plain pass ×0.72 — m3 «короткий и плотный», поэтому фиксированные накладные новых форматов видны заметнее. + +## 6. Стенд для повторных замеров и трейсов + +### 6.1. Что нужно зафиксировать + +1. Один и тот же бенчмарк-cpp на всех коммитах. Текущий `pp/wal/benchmarks/scraper_benchmark.cpp` уже идеально подходит — он есть в исходном виде в каждом из 8 коммитов после `3f0b2bd72`. Чтобы не зависеть от его эволюции, копируем последнюю версию (с 2-shards read) во временное место и применяем перед сборкой каждой ревизии. +2. Один и тот же набор фикстур. Сейчас в репо лежит только `pp/kube-api.metrics`. Для статьи минимум 2 файла: + - «cehp-runner-like» — длинные labels, разнообразные значения (можно использовать текущий `kube-api.metrics`). + - «m3-like» — плотный, в основном integer counters (можно сделать дамп с `node_exporter` или `cadvisor`, либо синтезировать). + - Опционально третий — пограничный (много float-ов с timestamp'ами). +3. `-c opt --copt=-march=native`, без ASan, без TSan. `pp-workflow.mdc` явно требует `-c opt` для перфметрик. +4. Tracy уже завязан в код (`profiling/profiling.h`, `ZoneScoped` в обеих бенч-функциях). Достаточно собрать с `--//bazel/toolchain:profiling=1` и подключиться `tracy-profiler`-ом. + +### 6.2. Скрипт прогона по ревизиям (черновик) + +```bash +# bench_sweep.sh — запускать из /prompp/pp +set -euo pipefail +COMMITS=( + 3f0b2bd72 # baseline + 4656f7137 # initial encoding (POC merged) + 0027219f0 # read + passing + 9b62e6ae # read/write opt + aca48508 # sample opt + 26d0af47 # 2 shards bench + 2036d7ea # parse_metric method + 1113d04b # parse_metric refactor +) + +FIXTURES=( + "cehp:performance_tests/test_data/cehp.metrics" + "m3:performance_tests/test_data/m3.metrics" +) + +mkdir -p bench_out +cp wal/benchmarks/scraper_benchmark.cpp /tmp/scraper_benchmark.cpp.frozen + +for h in "${COMMITS[@]}"; do + git checkout -q "$h" + cp /tmp/scraper_benchmark.cpp.frozen wal/benchmarks/scraper_benchmark.cpp + + bazel build -c opt --copt=-march=native //wal/benchmarks:scraper + + for kv in "${FIXTURES[@]}"; do + name="${kv%%:*}"; file="${kv##*:}" + out="bench_out/${h}_${name}.json" + ./bazel-bin/wal/benchmarks/scraper \ + --benchmark_repetitions=10 \ + --benchmark_min_time=1s \ + --benchmark_time_unit=ns \ + --benchmark_format=json \ + --benchmark_out="$out" \ + --benchmark_context=prom_scraper_file="$file" \ + --benchmark_filter='Parser|ScraperParse|ScraperRead' + done + + git checkout -q -- wal/benchmarks/scraper_benchmark.cpp +done + +git checkout -q - +``` + +Парсилка JSON-ов в одну CSV — отдельный шаг (`benchmark_repetitions=10` + `aggregate_name == "min"`). + +### 6.3. Tracy-прогоны для статьи + +| Ревизия | Что показать | Идея для скриншота | +|---|---|---| +| `3f0b2bd72` | baseline | один большой `ZoneScoped` в `ScraperParse`, плоско | +| `4656f7137` | POC: память упала, скорость просела | те же зоны, но видно широкие зелёные интервалы внутри `parse_metric` (ветвление в `add_label` / `add_sample`) | +| `0027219f0` | read заработал, но медленный | `ScraperRead` зона: показать дорогой `memcpy` с переменной длиной | +| `9b62e6ae` | главный win-back | те же `read_label` / `add_label` зоны, но в 2–3× уже | +| `2036d7ea` | inlining | пропал отдельный кадр конструктора `MetricParser` | + +Сборка под Tracy: +```bash +bazel build -c opt --copt=-march=native --//bazel/toolchain:profiling=1 //wal/benchmarks:scraper +``` + +В коде уже стоят `ZoneScoped` в `Parser`/`ScraperParse`/`ScraperRead`, и +есть закомментированные `ZoneScopedN("Scraper::parse")` / +`ZoneScopedN("MetricParser::parse")` / `ZoneScopedN("Metric::read()")` в +`scraper.h` (видно в `9b62e6ae` / `aca48508` диффах). Для трейсов их +нужно включить локально — это даст разбивку по фазам. + +Полезно ещё добавить более тонкие зоны (на время статьи, не в master): + +- внутри `parse_metric`: `ZoneScopedN("collect_labels")`, `ZoneScopedN("sort+hash")`, `ZoneScopedN("encode_count")`, `ZoneScopedN("encode_labels")`, `ZoneScopedN("encode_sample")`; +- внутри `Metric::read`: `ZoneScopedN("decode_count")`, `ZoneScopedN("decode_labels")`, `ZoneScopedN("decode_sample")`. + +С такими зонами трейс на 9b62e6ae против 0027219f0 наглядно покажет, +что выигрыш сидит именно в `decode_labels` и `encode_labels`/`encode_sample`. + +### 6.4. Валидация корректности на каждом шаге + +`pp-testing.mdc` требует ASan-прогон тестов перед мерджем; для статьи +это тоже подстраховка от скрытых багов в записи/чтении формата: + +```bash +bazel test -c dbg --asan //wal/hashdex/scraper:scraper_test +``` + +Для `4656f7137` тесты падают (по описанию коммита, тесты «прошли все» +только в `0027219f0`). Это можно использовать честно — «после POC +тесты красные, мы сначала делаем рабочую сериализацию, потом +разгоняем». + +### 6.5. Что ещё имеет смысл достать перед публикацией + +- **Распределение размеров полей** на тестовых дампах: гистограмма `name.length`, `value.length`, `offset_relative_to_metric` — чтобы доказать читателю гипотезу «всё умещается в 1–2 байта». Это `xxd`/python-скрипт по файлу. +- **Распределение sample-типов**: сколько uint8/uint16/uint32/double/NaN. Это и обоснует выбор маркеров. +- **Ratio**: средний размер метрики до/после в байтах. Можно прямо в бенчмарке вывести `bytes_count() / metric_count`. + +## 7. CSV из исходных замеров (нормализованная) + +Колонки: `Parse` — `BenchmarkScraperParse, ns`; `Pass` — `Plain Scraper Time, ns` (parse - parser); `Read` — `BenchmarkScraperRead, ns`; `Alloc` — `ScraperAllocMem, Mi`. В скобках — отношение к baseline. + +### Серия 1 (baseline `3f0b2bd72`) + +#### cehp-runner + +| Commit | Key change | Parse ns (×) | Pass ns (×) | Read ns (×) | Alloc Mi (×) | +|---|---|---:|---:|---:|---:| +| `3f0b2bd72` | baseline | 38171007 (1.00) | 27080951 (1.00) | 10927821 (1.00) | 13.746 (1.00) | +| `labels varint` | labels varint | 45775620 (0.83) | n/a | n/a | 5.01953 (2.74) | +| `sample encoding` | sample encoding | 47803708 (0.80) | n/a | n/a | 3.76953 (3.65) | +| `4656f7137` | initial encoding | 47564602 (0.80) | n/a | n/a | 4.14844 (3.31) | +| `0027219f0` | reading + extra offset added | 50035042 (0.76) | 38723316 (0.70) | 16912478 (0.65) | 4.12891 (3.33) | +| `9b62e6ae` | r/w optimizations | 46106593 (0.83) | 34340283 (0.79) | 7887484 (1.39) | 4.12891 (3.33) | +| `aca48508` | sample opt | 44739040 (0.85) | 33586988 (0.81) | 8198244 (1.33) | 4.12891 (3.33) | + +#### m3 + +| Commit | Key change | Parse ns (×) | Pass ns (×) | Read ns (×) | Alloc Mi (×) | +|---|---|---:|---:|---:|---:| +| `3f0b2bd72` | baseline | 22774389 (1.00) | 11749549 (1.00) | 1791641 (1.00) | 13.746 (1.00) | +| `labels varint` | labels varint | 28791658 (0.79) | n/a | n/a | 5.01953 (2.74) | +| `sample encoding` | sample encoding | 31195012 (0.73) | n/a | n/a | 3.76953 (3.65) | +| `4656f7137` | initial encoding | 28740757 (0.79) | n/a | n/a | 4.14844 (3.31) | +| `0027219f0` | reading + 2-shards bench update | 26255763 (0.87) | 14931157 (0.79) | 4980513 (0.36) | 4.12891 (3.33) | +| `9b62e6ae` | r/w optimizations | 28865275 (0.79) | 16914917 (0.69) | 1287539 (1.39) | 4.12891 (3.33) | +| `aca48508` | sample opt | 28900335 (0.79) | 16259238 (0.72) | 1265548 (1.42) | 4.12891 (3.33) | + +### Серия 2 (новый baseline `3f0b2bd72` после `26d0af47`) + +#### cehp-runner + +| Commit | Key change | Parse ns (×) | Pass ns (×) | Read ns (×) | Alloc Mi (×) | +|---|---|---:|---:|---:|---:| +| `3f0b2bd72` | baseline | 43942332 (1.00) | 32910757 (1.00) | 6080402 (1.00) | 13.746 (1.00) | +| `26d0af47` | 2 shards bench / state after sample opt | 45321131 (0.97) | 34085663 (0.97) | 3535429 (1.72) | 4.129 (3.33) | +| `2036d7ea` | MetricParser → parse_metric method | 44676447 (0.98) | 33235111 (0.99) | 3896916 (1.56) | 4.129 (3.33) | +| `1113d04b` | parse_metric refactoring | 45107031 (0.97) | 33783974 (0.97) | 3791985 (1.60) | 4.129 (3.33) | + +#### m3 + +| Commit | Key change | Parse ns (×) | Pass ns (×) | Read ns (×) | Alloc Mi (×) | +|---|---|---:|---:|---:|---:| +| `3f0b2bd72` | baseline | 22985376 (1.00) | 11277242 (1.00) | 1245206 (1.00) | 13.746 (1.00) | +| `26d0af47` | 2 shards bench / state after sample opt | 29213092 (0.79) | 17359001 (0.65) | 796406 (1.56) | 4.129 (3.33) | +| `2036d7ea` | MetricParser → parse_metric method | 27959700 (0.82) | 16263446 (0.69) | 918037 (1.36) | 4.129 (3.33) | +| `1113d04b` | parse_metric refactoring | 28861682 (0.80) | 17450605 (0.65) | 810836 (1.54) | 4.129 (3.33) | + +## 8. Open questions / TODO перед публикацией + +- [ ] Воспроизвести замеры на свежей машине, на которой будут писаться остальные графики статьи (одна и та же CPU, одна и та же сборка `-c opt --copt=-march=native`). +- [ ] Создать/добавить в репо два публикуемых дампа фикстур (`cehp.metrics`, `m3.metrics`) либо описать, как их получить (например, `curl http://kube-apiserver:443/metrics`). +- [ ] Снять Tracy-трейсы на 5 ревизиях из таблицы 6.3. +- [ ] Сделать гистограммы распределения `name.length` / `value.length` / `value type` на двух фикстурах. +- [ ] Прогнать `bazel test -c dbg --asan //wal/hashdex/scraper:scraper_test` на каждой ревизии, отметить, начиная с какой проходит. From 032fc4150e840371b607a618307247dc1fe94c93 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 13 May 2026 10:04:56 +0000 Subject: [PATCH 42/48] scraper 3f0b2bd72 --- pp/wal/benchmarks/scraper_benchmark.cpp | 2 + pp/wal/hashdex/scraper/encoding.h | 370 --------------- pp/wal/hashdex/scraper/encoding_tests.cpp | 209 --------- pp/wal/hashdex/scraper/marked.h | 218 --------- pp/wal/hashdex/scraper/marked_common.h | 59 --- pp/wal/hashdex/scraper/parser.h | 13 +- pp/wal/hashdex/scraper/scraper.h | 534 ++++++++++++++-------- 7 files changed, 354 insertions(+), 1051 deletions(-) delete mode 100644 pp/wal/hashdex/scraper/encoding.h delete mode 100644 pp/wal/hashdex/scraper/encoding_tests.cpp delete mode 100644 pp/wal/hashdex/scraper/marked.h delete mode 100644 pp/wal/hashdex/scraper/marked_common.h diff --git a/pp/wal/benchmarks/scraper_benchmark.cpp b/pp/wal/benchmarks/scraper_benchmark.cpp index 81498a7497..1a3f09177e 100644 --- a/pp/wal/benchmarks/scraper_benchmark.cpp +++ b/pp/wal/benchmarks/scraper_benchmark.cpp @@ -4,6 +4,8 @@ #include "benchmark/statistic.h" #include "primitives/timeseries.h" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Warray-bounds" #include "profiling/profiling.h" #include "wal/hashdex/scraper/scraper.h" diff --git a/pp/wal/hashdex/scraper/encoding.h b/pp/wal/hashdex/scraper/encoding.h deleted file mode 100644 index 9dedf4c7ba..0000000000 --- a/pp/wal/hashdex/scraper/encoding.h +++ /dev/null @@ -1,370 +0,0 @@ -#pragma once - -#include -#include - -#include "bare_bones/bit.h" -#include "marked_common.h" -#include "primitives/sample.h" -#include "prometheus/value.h" - -namespace PromPP::WAL::hashdex::scraper::encoding { -enum class SampleValueType : uint8_t { kUint32 = 0b0000'0000, kDouble, kUint8, kUint16, kFloat, kZero, kNaN }; - -struct LayoutMarker { - bool has_ts : 1; - uint8_t length_bytes : 3; - SampleValueType sample_value_type : 4; - - [[nodiscard]] PROMPP_LAMBDA_INLINE bool has_timestamp() const noexcept { return has_ts; } - - [[nodiscard]] PROMPP_LAMBDA_INLINE uint8_t size_length_in_bytes() const noexcept { return length_bytes; } - - [[nodiscard]] PROMPP_LAMBDA_INLINE SampleValueType value_type() const noexcept { return sample_value_type; } - - static PROMPP_LAMBDA_INLINE LayoutMarker make(bool has_ts, uint32_t labels_count, SampleValueType value_type) noexcept { - const uint8_t bytes_for_count = BareBones::Bit::to_ceil_bytes(std::bit_width(labels_count)); - return LayoutMarker{.has_ts = has_ts, .length_bytes = bytes_for_count, .sample_value_type = value_type}; - } -}; - -class SampleCodec { - public: - static constexpr size_t kMaximumEncodingSize = sizeof(Primitives::Sample); - - static char* encode(char* out, const LayoutMarker layout, Primitives::Sample sample) { - using encoding::SampleValueType; - - const double val = sample.value(); - if (const auto type = layout.value_type(); type == SampleValueType::kUint32) [[likely]] { - out = write_value(out, static_cast(val)); - } else if (type == SampleValueType::kDouble) [[likely]] { - out = write_value(out, val); - } else if (type == SampleValueType::kUint8) [[unlikely]] { - out = write_value(out, static_cast(val)); - } else if (type == SampleValueType::kUint16) [[unlikely]] { - out = write_value(out, static_cast(val)); - } else if (type == SampleValueType::kFloat) [[unlikely]] { - out = write_value(out, static_cast(val)); - } - - if (layout.has_timestamp()) [[unlikely]] { - out = write_value(out, sample.timestamp()); - } - - return out; - } - - struct DecodeResult { - const char* next; - Primitives::Sample sample; - }; - - static DecodeResult decode(const char* in, const LayoutMarker layout, int64_t default_ts) { - using encoding::SampleValueType; - - DecodeResult result{}; - - double& val = result.sample.value(); - uint64_t chunk; - std::memcpy(&chunk, in, sizeof(chunk)); - - if (const auto type = layout.value_type(); type == SampleValueType::kUint32) [[likely]] { - val = static_cast(static_cast(chunk)); - in += sizeof(uint32_t); - } else if (type == SampleValueType::kDouble) [[likely]] { - val = std::bit_cast(chunk); - in += sizeof(double); - } else if (type == SampleValueType::kUint8) [[unlikely]] { - val = static_cast(static_cast(chunk)); - in += sizeof(uint8_t); - } else if (type == SampleValueType::kUint16) [[unlikely]] { - val = static_cast(static_cast(chunk)); - in += sizeof(uint16_t); - } else if (type == SampleValueType::kFloat) [[unlikely]] { - val = static_cast(std::bit_cast(static_cast(chunk))); - in += sizeof(float); - } else if (type == SampleValueType::kZero) [[unlikely]] { - val = 0.0; - } else { - val = Prometheus::kNormalNan; - } - - if (auto& timestamp = result.sample.timestamp(); layout.has_timestamp()) [[unlikely]] { - std::memcpy(×tamp, in, sizeof(timestamp)); - in += sizeof(timestamp); - } else { - timestamp = default_ts; - } - - result.next = in; - - return result; - } - - [[nodiscard]] static SampleValueType value_type(const double val) noexcept { - if (std::isnan(val)) [[unlikely]] { - return SampleValueType::kNaN; - } - - if (val == 0.0) [[unlikely]] { - return SampleValueType::kZero; - } - - if (std::trunc(val) == val && val > 0.0 && val <= std::numeric_limits::max()) [[likely]] { - const auto uval = static_cast(val); - if (uval <= std::numeric_limits::max()) { - return SampleValueType::kUint8; - } - if (uval <= std::numeric_limits::max()) { - return SampleValueType::kUint16; - } - return SampleValueType::kUint32; - } - - if (const auto f = static_cast(val); static_cast(f) == val) [[unlikely]] { - return SampleValueType::kFloat; - } - - return SampleValueType::kDouble; - } - - private: - template - PROMPP_ALWAYS_INLINE static char* write_value(char* out, const T& val) noexcept { - std::memcpy(out, &val, sizeof(T)); - return out + sizeof(T); - } -}; - -class LabelCodec { - public: - static constexpr size_t kMaximumEncodingSize = sizeof(uint8_t) + 4 * sizeof(uint32_t); - - static char* encode(char* out, const MarkedLabel label) noexcept { - if (label.name.offset == 0 && label.name.length == 0) [[likely]] { - return encode_value_only(out, label.value.offset, label.value.length); - } - - if ((label.name.offset | label.name.length | label.value.offset | label.value.length) <= 0xFF) [[likely]] { - return encode_4_bytes(out, label.name.offset, label.name.length, label.value.offset, label.value.length); - } - - return encode_generic(out, label.name.offset, label.name.length, label.value.offset, label.value.length); - } - - struct DecodeResult { - const char* next; - MarkedLabel label; - }; - - static DecodeResult decode(const char* in) noexcept { - uint64_t chunk; - std::memcpy(&chunk, in, sizeof(chunk)); - const auto layout = static_cast(chunk); - - if (layout == 0b01010101) [[likely]] { - return decode_4_bytes(in, chunk); - } - - if ((layout & 0x0F) == 0) [[likely]] { - return decode_value_only(in, chunk, layout); - } - - return decode_generic(++in, layout); - } - - private: - static PROMPP_ALWAYS_INLINE char* encode_value_only(char* out, const uint32_t label_value_offset, const uint32_t label_value_length) noexcept { - char* start = out++; - - const uint8_t sz2 = push_and_encode(out, label_value_offset); - out += szm_[sz2]; - const uint8_t sz3 = push_and_encode(out, label_value_length); - out += szm_[sz3]; - - *start = (sz2 << 4) | (sz3 << 6); - - return out; - } - - static PROMPP_ALWAYS_INLINE char* encode_4_bytes(char* out, - const uint8_t label_name_offset, - const uint8_t label_name_length, - const uint8_t label_value_offset, - const uint8_t label_value_length) noexcept { - const uint64_t chunk = static_cast(0b01010101) | static_cast(label_name_offset) << 8 | static_cast(label_name_length) << 16 | - static_cast(label_value_offset) << 24 | static_cast(label_value_length) << 32; - - std::memcpy(out, &chunk, sizeof(chunk)); - return out + 5; - } - - static PROMPP_ALWAYS_INLINE char* encode_generic(char* out, - const uint32_t label_name_offset, - const uint32_t label_name_length, - const uint32_t label_value_offset, - const uint32_t label_value_length) noexcept { - char* start = out++; - const uint8_t sz0 = push_and_encode(out, label_name_offset); - out += szm_[sz0]; - const uint8_t sz1 = push_and_encode(out, label_name_length); - out += szm_[sz1]; - const uint8_t sz2 = push_and_encode(out, label_value_offset); - out += szm_[sz2]; - const uint8_t sz3 = push_and_encode(out, label_value_length); - out += szm_[sz3]; - - *start = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); - - return out; - } - - static PROMPP_ALWAYS_INLINE DecodeResult decode_4_bytes(const char* in, const uint64_t chunk) noexcept { - const auto name_off = static_cast(chunk >> 8); - const auto name_len = static_cast(chunk >> 16); - const auto value_off = static_cast(chunk >> 24); - const auto value_len = static_cast(chunk >> 32); - - return DecodeResult{.next = in + 5, .label = {.name = {.offset = name_off, .length = name_len}, .value = {.offset = value_off, .length = value_len}}}; - } - - static PROMPP_ALWAYS_INLINE DecodeResult decode_value_only(const char* in, uint64_t chunk, const uint8_t layout) noexcept { - const uint8_t sz2 = (layout >> 4) & 0b11; - const uint8_t sz3 = (layout >> 6) & 0b11; - - chunk >>= 8; - size_t used = 1; - - uint32_t value_off = 0; - uint32_t value_len = 0; - - if (sz2 == 0b01) [[likely]] { - value_off = static_cast(chunk); - used += sizeof(uint8_t); - chunk >>= BareBones::Bit::to_bits(sizeof(uint8_t)); - } else if (sz2 == 0b10) [[unlikely]] { - value_off = static_cast(chunk); - used += sizeof(uint16_t); - chunk >>= BareBones::Bit::to_bits(sizeof(uint16_t)); - } else if (sz2 == 0b11) [[unlikely]] { - value_off = static_cast(chunk); - used += sizeof(uint32_t); - chunk >>= BareBones::Bit::to_bits(sizeof(uint32_t)); - } - if (sz3 == 0b01) [[likely]] { - value_len = static_cast(chunk); - used += sizeof(uint8_t); - } else if (sz3 == 0b10) [[unlikely]] { - value_len = static_cast(chunk); - used += sizeof(uint16_t); - } else if (sz3 == 0b11) [[unlikely]] { - if (used + sizeof(uint32_t) <= sizeof(chunk)) [[likely]] { - value_len = static_cast(chunk); - } else [[unlikely]] { - std::memcpy(&value_len, in + used, sizeof(value_len)); - } - used += sizeof(value_len); - } - - return DecodeResult{.next = in + used, .label = {.name = {.offset = 0, .length = 0}, .value = {.offset = value_off, .length = value_len}}}; - } - - static PROMPP_ALWAYS_INLINE DecodeResult decode_generic(const char* in, const uint8_t layout) noexcept { - const uint8_t sz0 = layout & 0b11; - const uint8_t sz1 = (layout >> 2) & 0b11; - const uint8_t sz2 = (layout >> 4) & 0b11; - const uint8_t sz3 = (layout >> 6) & 0b11; - - const uint32_t name_off = read_val_partial(in, sz0); - const uint32_t name_len = read_val_partial(in, sz1); - const uint32_t value_off = read_val_partial(in, sz2); - const uint32_t value_len = read_val_partial(in, sz3); - - return DecodeResult{.next = in, .label = {.name = {.offset = name_off, .length = name_len}, .value = {.offset = value_off, .length = value_len}}}; - } - - static PROMPP_ALWAYS_INLINE uint8_t push_and_encode(char* out, uint32_t v) noexcept { - if (v == 0) [[unlikely]] { - return 0b00; - } - std::memcpy(out, &v, sizeof(v)); - if (v <= 0xFF) [[likely]] { - return 0b01; - } - if (v <= 0xFFFF) [[unlikely]] { - return 0b10; - } - - return 0b11; - } - - template - static PROMPP_ALWAYS_INLINE uint32_t read_val_typed(const char*& p) noexcept { - T v; - std::memcpy(&v, p, sizeof(T)); - p += sizeof(T); - return static_cast(v); - } - - static PROMPP_ALWAYS_INLINE uint32_t read_val_partial(const char*& p, uint8_t sz) noexcept { - switch (sz) { - case 0b00: { - return 0; - } - case 0b01: { - return read_val_typed(p); - } - case 0b10: { - return read_val_typed(p); - } - default: { - return read_val_typed(p); - } - } - } - - static constexpr uint8_t szm_[4] = {0, 1, 2, 4}; -}; - -class LayoutCountCodec { - public: - static constexpr size_t kMaximumEncodingSize = sizeof(uint64_t); - - static PROMPP_ALWAYS_INLINE char* encode(char* out, const LayoutMarker layout, const uint32_t count) noexcept { - uint64_t chunk = 0; - std::memcpy(&chunk, &layout, sizeof(layout)); - chunk |= static_cast(count) << 8; - std::memcpy(out, &chunk, sizeof(chunk)); - - const uint32_t bytes_written = sizeof(layout) + layout.size_length_in_bytes(); - return out + bytes_written; - } - - struct DecodeResult { - const char* next; - LayoutMarker layout; - uint32_t count; - }; - - static PROMPP_ALWAYS_INLINE DecodeResult decode(const char* in) noexcept { - uint64_t chunk; - std::memcpy(&chunk, in, sizeof(chunk)); - - LayoutMarker layout{}; - std::memcpy(&layout, &chunk, sizeof(layout)); - - chunk >>= 8; - const uint64_t mask = (1ULL << BareBones::Bit::to_bits(layout.size_length_in_bytes())) - 1; - - auto labels_count = static_cast(chunk & mask); - - return {in + sizeof(layout) + layout.size_length_in_bytes(), layout, labels_count}; - } -}; - -static constexpr uint32_t metric_maximum_encoding_size(uint32_t labels_count) noexcept { - return LayoutCountCodec::kMaximumEncodingSize + LabelCodec::kMaximumEncodingSize * labels_count + SampleCodec::kMaximumEncodingSize; -} - -} // namespace PromPP::WAL::hashdex::scraper::encoding \ No newline at end of file diff --git a/pp/wal/hashdex/scraper/encoding_tests.cpp b/pp/wal/hashdex/scraper/encoding_tests.cpp deleted file mode 100644 index 6a5d021e5b..0000000000 --- a/pp/wal/hashdex/scraper/encoding_tests.cpp +++ /dev/null @@ -1,209 +0,0 @@ -#include - -#include -#include -#include -#include - -#include "encoding.h" -#include "primitives/sample.h" -#include "prometheus/value.h" - -namespace { - -using PromPP::Primitives::Sample; -using PromPP::Primitives::Timestamp; -using PromPP::WAL::hashdex::scraper::encoding::LabelCodec; -using PromPP::WAL::hashdex::scraper::encoding::LayoutMarker; -using PromPP::WAL::hashdex::scraper::encoding::SampleCodec; -using PromPP::WAL::hashdex::scraper::encoding::SampleValueType; - -struct ValueTypeCase { - double input; - SampleValueType expected; -}; - -class ValueTypeFixture : public testing::TestWithParam {}; - -TEST_P(ValueTypeFixture, ClassifyValueCorrectly) { - // Arrange - - // Act - const auto actual = SampleCodec::value_type(GetParam().input); - - // Assert - EXPECT_EQ(actual, GetParam().expected); -} - -INSTANTIATE_TEST_SUITE_P(ValueTypeTests, - ValueTypeFixture, - testing::Values(ValueTypeCase{.input = PromPP::Prometheus::kNormalNan, .expected = SampleValueType::kNaN}, - ValueTypeCase{.input = PromPP::Prometheus::kStaleNan, .expected = SampleValueType::kNaN}, - ValueTypeCase{.input = std::numeric_limits::quiet_NaN(), .expected = SampleValueType::kNaN}, - - ValueTypeCase{.input = 0.0, .expected = SampleValueType::kZero}, - ValueTypeCase{.input = -0.0, .expected = SampleValueType::kZero}, - - ValueTypeCase{.input = 1.0, .expected = SampleValueType::kUint8}, - ValueTypeCase{.input = 255.0, .expected = SampleValueType::kUint8}, - - ValueTypeCase{.input = 256.0, .expected = SampleValueType::kUint16}, - ValueTypeCase{.input = 65535.0, .expected = SampleValueType::kUint16}, - - ValueTypeCase{.input = 65536.0, .expected = SampleValueType::kUint32}, - ValueTypeCase{.input = 4294967295.0, .expected = SampleValueType::kUint32}, - - ValueTypeCase{.input = 3.5, .expected = SampleValueType::kFloat}, - ValueTypeCase{.input = -42.0, .expected = SampleValueType::kFloat}, - - ValueTypeCase{.input = 1e-10, .expected = SampleValueType::kDouble}, - ValueTypeCase{.input = 3.141592653589793, .expected = SampleValueType::kDouble}, - ValueTypeCase{.input = 0.1, .expected = SampleValueType::kDouble})); - -struct SampleCodecCase { - Sample sample; - bool has_ts; -}; - -class SampleCodecFixture : public testing::TestWithParam { - protected: - static constexpr size_t kBufSize = 64; - - SampleCodec::DecodeResult encode_and_decode(const LayoutMarker layout, Sample sample, int64_t default_ts = -1) { - buf_.fill(0); - char* start = buf_.data(); - char* end = SampleCodec::encode(start, layout, sample); - - const auto res = SampleCodec::decode(start, layout, default_ts); - - EXPECT_EQ(res.next, end); - return res; - } - - std::array buf_{}; - const Timestamp default_ts_ = -1; -}; - -TEST_P(SampleCodecFixture, CorrectSample) { - // Arrange - const auto layout = LayoutMarker::make(GetParam().has_ts, 0, SampleCodec::value_type(GetParam().sample.value())); - - Sample expected = GetParam().sample; - if (!GetParam().has_ts) { - expected.timestamp() = default_ts_; - } - - // Act - const auto res = encode_and_decode(layout, GetParam().sample, default_ts_); - - // Assert - EXPECT_EQ(res.sample, expected); -} - -INSTANTIATE_TEST_SUITE_P(SampleCodecTests, - SampleCodecFixture, - testing::Values(SampleCodecCase{.sample = Sample(42, 123.0), .has_ts = true}, - SampleCodecCase{.sample = Sample(12345, std::numbers::pi), .has_ts = false}, - SampleCodecCase{.sample = Sample(111, 1.2345), .has_ts = true}, - SampleCodecCase{.sample = Sample(7, 255.0), .has_ts = true}, - SampleCodecCase{.sample = Sample(9, 65535.0), .has_ts = true}, - SampleCodecCase{.sample = Sample(222, 0.0), .has_ts = true}, - SampleCodecCase{.sample = Sample(333, PromPP::Prometheus::kNormalNan), .has_ts = true}, - SampleCodecCase{.sample = Sample(1000, 42.0), .has_ts = false}, - SampleCodecCase{.sample = Sample(2048, 4242.0), .has_ts = true})); - -class LabelCodecFixture : public testing::Test { - protected: - static constexpr size_t kBufSize = 64; - std::array buf_{}; - - LabelCodec::DecodeResult encode_and_decode(uint32_t name_off, uint32_t name_len, uint32_t value_off, uint32_t value_len) { - buf_.fill(0); - char* start = buf_.data(); - char* end = LabelCodec::encode(start, PromPP::WAL::hashdex::scraper::MarkedLabel{.name = {.offset = name_off, .length = name_len}, - .value = {.offset = value_off, .length = value_len}}); - - const auto res = LabelCodec::decode(start); - - EXPECT_EQ(res.next, end); - return res; - } -}; - -TEST_F(LabelCodecFixture, FastPath_Layout01010101) { - // Arrange - buf_.fill(0); - char* start = buf_.data(); - start[0] = static_cast(0b01010101); - start[1] = 10; - start[2] = 11; - start[3] = 12; - start[4] = 13; - - // Act - const auto res = LabelCodec::decode(buf_.data()); - - // Assert - EXPECT_EQ(res.label.name.offset, 10u); - EXPECT_EQ(res.label.name.length, 11u); - EXPECT_EQ(res.label.value.offset, 12u); - EXPECT_EQ(res.label.value.length, 13u); - EXPECT_EQ(res.next, buf_.data() + 5); -} - -TEST_F(LabelCodecFixture, SimplifiedPath_NameFieldsZero) { - // Arrange - buf_.fill(0); - uint8_t layout = 0b10010000; - buf_[0] = static_cast(layout); - buf_[1] = 42; - uint16_t len = 1234; - std::memcpy(buf_.data() + 2, &len, 2); - - // Act - const auto res = LabelCodec::decode(buf_.data()); - - // Assert - EXPECT_EQ(res.label.name.offset, 0u); - EXPECT_EQ(res.label.name.length, 0u); - EXPECT_EQ(res.label.value.offset, 42u); - EXPECT_EQ(res.label.value.length, 1234u); - EXPECT_EQ(res.next, buf_.data() + 1 + 1 + 2); -} - -struct LabelCase { - uint32_t name_off; - uint32_t name_len; - uint32_t value_off; - uint32_t value_len; -}; - -class LabelCodecParamFixture : public LabelCodecFixture, public testing::WithParamInterface {}; - -TEST_P(LabelCodecParamFixture, EncodeDecode) { - // Arrange - - // Act - const auto res = encode_and_decode(GetParam().name_off, GetParam().name_len, GetParam().value_off, GetParam().value_len); - - // Assert - EXPECT_EQ(res.label.name.offset, GetParam().name_off); - EXPECT_EQ(res.label.name.length, GetParam().name_len); - EXPECT_EQ(res.label.value.offset, GetParam().value_off); - EXPECT_EQ(res.label.value.length, GetParam().value_len); -} - -INSTANTIATE_TEST_SUITE_P(LabelCodecTests, - LabelCodecParamFixture, - testing::Values(LabelCase{.name_off = 0, .name_len = 0, .value_off = 0, .value_len = 0}, - LabelCase{.name_off = 1, .name_len = 2, .value_off = 3, .value_len = 4}, - LabelCase{.name_off = 300, .name_len = 400, .value_off = 500, .value_len = 600}, - LabelCase{.name_off = 100000, .name_len = 200000, .value_off = 300000, .value_len = 400000}, - LabelCase{.name_off = 0, .name_len = 0, .value_off = 300000, .value_len = 400000}, - LabelCase{.name_off = 0, .name_len = 0, .value_off = 1234, .value_len = 123456}, - LabelCase{.name_off = 0, .name_len = 12, .value_off = 1234, .value_len = 123456}, - LabelCase{.name_off = 0, .name_len = 12, .value_off = 1234, .value_len = 255}, - LabelCase{.name_off = 0, .name_len = 12, .value_off = 128, .value_len = 2355}, - LabelCase{.name_off = 256, .name_len = 128, .value_off = 255, .value_len = 255})); - -} // namespace \ No newline at end of file diff --git a/pp/wal/hashdex/scraper/marked.h b/pp/wal/hashdex/scraper/marked.h deleted file mode 100644 index b3c180388e..0000000000 --- a/pp/wal/hashdex/scraper/marked.h +++ /dev/null @@ -1,218 +0,0 @@ -#pragma once - -#include - -#include "bare_bones/vector.h" -#include "encoding.h" -#include "marked_common.h" -#include "primitives/primitives.h" -#include "prometheus/metric.h" - -namespace PromPP::WAL::hashdex::scraper::inline marked { - -class Metric { - public: - using MarkedT = MarkedMetric; - - struct Context { - std::string_view buffer; - const BareBones::Memory& bytes_buffer; - Primitives::Timestamp default_timestamp{}; - }; - - Metric(const Context& ctx, const MarkedMetric* item) - : buffer_(ctx.buffer), bytes_buffer_(ctx.bytes_buffer), item_(item), default_timestamp_(ctx.default_timestamp) {} - - [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetric* item() const noexcept { return item_; } - PROMPP_ALWAYS_INLINE void set_item(const MarkedMetric* item) noexcept { item_ = item; } - - [[nodiscard]] PROMPP_ALWAYS_INLINE uint64_t hash() const noexcept { return item_->hash; } - - template - void read(Timeseries& ts) const { - const char* ptr = bytes_buffer_.control_block().data + item_->data_offset; - - const auto [next_ptr, layout, labels_count] = encoding::LayoutCountCodec::decode(ptr); - ptr = next_ptr; - - ts.label_set().resize(labels_count); - - auto label_iter = ts.label_set().begin(); - for (uint32_t i = 0; i < labels_count; ++i) { - const auto [next_ptr, label] = encoding::LabelCodec::decode(ptr); - ptr = next_ptr; - - if (const auto buf_ptr = buffer_.data() + item_->base_offset; label.name.is_reserved_name()) [[unlikely]] { - std::construct_at(label_iter++, Prometheus::kMetricLabelName, std::string_view(buf_ptr + label.value.offset, label.value.length)); - } else { - std::construct_at(label_iter++, std::string_view(buf_ptr + label.name.offset, label.name.length), - std::string_view(buf_ptr + label.value.offset, label.value.length)); - } - } - - auto [p, sample] = encoding::SampleCodec::decode(ptr, layout, default_timestamp_); - - ts.samples().emplace_back(sample); - } - - private: - std::string_view buffer_; - const BareBones::Memory& bytes_buffer_; - const MarkedMetric* item_{}; - Primitives::Timestamp default_timestamp_; -}; - -class Metadata { - public: - using MarkedT = MarkedMetadata; - - struct Context { - std::string_view buffer; - }; - - Metadata(const Context& ctx, const MarkedMetadata* item) : buffer_(ctx.buffer), item_(item) {} - - [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetadata* item() const noexcept { return item_; } - PROMPP_ALWAYS_INLINE void set_item(const MarkedMetadata* item) noexcept { item_ = item; } - - [[nodiscard]] PROMPP_ALWAYS_INLINE Prometheus::MetadataType type() const noexcept { return item_->type; } - [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view metric_name() const noexcept { return item_->metric_name.view(buffer_); } - [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view text() const noexcept { return item_->text.view(buffer_); } - - private: - std::string_view buffer_; - const MarkedMetadata* item_{}; -}; - -template -class MarkupBuffer { - public: - using MarkedT = typename T::MarkedT; - using Context = typename T::Context; - - class IteratorSentinel {}; - - class Iterator { - public: - using iterator_category = std::forward_iterator_tag; - using value_type = T; - using difference_type = ptrdiff_t; - using pointer = value_type*; - using reference = value_type&; - - Iterator(const Context& ctx, const MarkedT* ptr, uint32_t items_count) : item_(ctx, ptr), ptr_(ptr), items_count_(items_count), ctx_(ctx) {} - - [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type& operator*() const noexcept { return item_; } - [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type* operator->() const noexcept { return &item_; } - - PROMPP_ALWAYS_INLINE Iterator& operator++() noexcept { - item_.set_item(++ptr_); - --items_count_; - return *this; - } - - PROMPP_ALWAYS_INLINE Iterator operator++(int) noexcept { - auto tmp = *this; - ++(*this); - return tmp; - } - - PROMPP_ALWAYS_INLINE bool operator==(const IteratorSentinel&) const noexcept { return items_count_ == 0; } - - private: - T item_; - const MarkedT* ptr_; - uint32_t items_count_; - Context ctx_; - }; - - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t items_count() const noexcept { return buffer_.size(); } - [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return buffer_.allocated_memory(); } - - protected: - BareBones::Vector buffer_; -}; - -class MetricMarkupBuffer : public MarkupBuffer { - public: - using Base = MarkupBuffer; - using Iterator = typename Base::Iterator; - using IteratorSentinel = typename Base::IteratorSentinel; - - [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer, Primitives::Timestamp default_ts) const noexcept { - return {typename Base::Context{buffer, bytes_buffer_, default_ts}, Base::buffer_.data(), Base::items_count()}; - } - [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } - - void bytes_enlarge(uint32_t extra_bytes) noexcept { - const uint32_t offset = bytes_count(); - - bytes_buffer_.grow_to_fit_at_least(offset + extra_bytes); - - bytes_ptr_ = bytes_buffer_.control_block().data + offset; - } - - [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return Base::buffer_.allocated_memory() + bytes_buffer_.allocated_memory(); } - - PROMPP_ALWAYS_INLINE void initialize(size_t reserve_bytes) noexcept { - this->buffer_.clear(); - - const size_t bytes_buffer_reserve = (reserve_bytes / 3) * 2; - const size_t items_buffer_reserve = (bytes_buffer_reserve / 3) / sizeof(MarkedMetric); - - this->buffer_.reserve(items_buffer_reserve); - bytes_buffer_.resize_to_fit_at_least(bytes_buffer_reserve); - bytes_ptr_ = bytes_buffer_.control_block().data; - } - - PROMPP_ALWAYS_INLINE void add_hash(uint64_t hash) noexcept { this->buffer_.back().hash = hash; } - - PROMPP_ALWAYS_INLINE void add_metric(uint32_t global_offset) noexcept { - this->buffer_.push_back(MarkedMetric{.hash = {}, .base_offset = global_offset, .data_offset = bytes_count()}); - } - - void add_layout_and_count(const encoding::LayoutMarker layout, const uint32_t count) noexcept { - bytes_ptr_ = encoding::LayoutCountCodec::encode(bytes_ptr_, layout, count); - } - - void add_label(MarkedLabel label) noexcept { bytes_ptr_ = encoding::LabelCodec::encode(bytes_ptr_, label); } - - void add_sample(encoding::LayoutMarker layout, const Primitives::Sample sample) noexcept { - bytes_ptr_ = encoding::SampleCodec::encode(bytes_ptr_, layout, sample); - } - - void add_padding() noexcept { - constexpr size_t kPaddingSizeBytes = 16; - bytes_enlarge(kPaddingSizeBytes); - } - - private: - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t bytes_count() const noexcept { return bytes_ptr_ - bytes_buffer_.control_block().data; } - - BareBones::Memory bytes_buffer_; - char* bytes_ptr_{}; -}; - -class MetadataMarkupBuffer : public MarkupBuffer { - public: - using Base = MarkupBuffer; - using Iterator = typename Base::Iterator; - using IteratorSentinel = typename Base::IteratorSentinel; - - [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer) const noexcept { - return {typename Base::Context{buffer}, this->buffer_.data(), this->items_count()}; - } - [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } - - PROMPP_ALWAYS_INLINE void initialize(size_t reserve_bytes) noexcept { - this->buffer_.clear(); - const size_t items_buffer_reserve = reserve_bytes / sizeof(typename Base::MarkedT); - this->buffer_.reserve(items_buffer_reserve); - } - - PROMPP_ALWAYS_INLINE void add(MarkedString metric_name, MarkedString text, Prometheus::MetadataType type) noexcept { - this->buffer_.emplace_back(metric_name, text, type); - } -}; - -} // namespace PromPP::WAL::hashdex::scraper::inline marked \ No newline at end of file diff --git a/pp/wal/hashdex/scraper/marked_common.h b/pp/wal/hashdex/scraper/marked_common.h deleted file mode 100644 index 8036173543..0000000000 --- a/pp/wal/hashdex/scraper/marked_common.h +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once - -#include - -#include "primitives/sample.h" -#include "prometheus/metric.h" -#include "prometheus/value.h" - -namespace PromPP::WAL::hashdex::scraper::inline marked { - -#pragma pack(push, 1) -struct MarkedString { - uint32_t offset = 0; - uint32_t length = 0; - - [[nodiscard]] PROMPP_ALWAYS_INLINE static MarkedString create(std::string_view value, std::string_view buffer) noexcept { - return { - .offset = static_cast(value.data() - buffer.data()), - .length = static_cast(value.size()), - }; - } - - [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_reserved_name() const noexcept { return offset == 0 && length == 0; } - - [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_empty() const noexcept { return length == 0; } - - [[nodiscard]] std::string_view view(const std::string_view& buffer) const noexcept { - if (is_reserved_name()) [[unlikely]] { - return Prometheus::kMetricLabelName; - } - - return buffer.substr(offset, length); - } -}; - -struct MarkedLabel { - MarkedString name{}; - MarkedString value; -}; - -struct MarkedSample { - Primitives::Sample sample{}; - bool has_ts{}; -}; - -struct MarkedMetric { - uint64_t hash; - uint32_t base_offset; - uint32_t data_offset; -}; - -struct MarkedMetadata { - MarkedString metric_name{}; - MarkedString text{}; - Prometheus::MetadataType type{}; -}; -#pragma pack(pop) - -} // namespace PromPP::WAL::hashdex::scraper::inline marked \ No newline at end of file diff --git a/pp/wal/hashdex/scraper/parser.h b/pp/wal/hashdex/scraper/parser.h index 76e03bdde6..c539cb5941 100644 --- a/pp/wal/hashdex/scraper/parser.h +++ b/pp/wal/hashdex/scraper/parser.h @@ -31,13 +31,13 @@ template } template -concept ParserInterface = requires(Parser& parser, const Parser& const_parser, Primitives::Timestamp& timestamp, bool& has_timestamp) { +concept ParserInterface = requires(Parser& parser, const Parser& const_parser, Primitives::Timestamp& timestamp) { { parser.tokenizer() }; { Prometheus::textparse::TokenizerInterface }; { Prometheus::textparse::TokenizerInterface }; { const_parser.is_value_token() } -> std::same_as; - { parser.parse_timestamp(timestamp, has_timestamp) } -> std::same_as; + { parser.parse_timestamp(timestamp) } -> std::same_as; { const_parser.validate_parse_result() } -> std::same_as; { const_parser.validate_parse_sample_result() } -> std::same_as; }; @@ -59,15 +59,13 @@ class PrometheusParser { return (tokenizer_.token() == Token::kLinebreak || tokenizer_.token() == Token::kEOF) ? Error::kNoError : Error::kUnexpectedToken; } - [[nodiscard]] Error parse_timestamp(Primitives::Timestamp& timestamp, bool& has_timestamp) noexcept { - has_timestamp = false; + [[nodiscard]] Error parse_timestamp(Primitives::Timestamp& timestamp) noexcept { if (is_timestamp_token()) { if (!parse_numeric_value(tokenizer_.token_str(), timestamp)) [[unlikely]] { return Error::kInvalidTimestamp; } tokenizer_.next_non_whitespace(); - has_timestamp = true; } return Error::kNoError; @@ -98,8 +96,7 @@ class OpenMetricsParser { return (tokenizer_.token() == Token::kLinebreak || tokenizer_.token() == Token::kExemplar) ? Error::kNoError : Error::kUnexpectedToken; } - [[nodiscard]] Error parse_timestamp(Primitives::Timestamp& timestamp, bool& has_timestamp) noexcept { - has_timestamp = false; + [[nodiscard]] Error parse_timestamp(Primitives::Timestamp& timestamp) noexcept { if (tokenizer_.token() == Token::kTimestamp) { if (double float_timestamp; parse_timestamp_as_float(float_timestamp)) [[likely]] { timestamp = static_cast(float_timestamp * 1000.0); @@ -108,8 +105,8 @@ class OpenMetricsParser { } tokenizer_.next_non_whitespace(); - has_timestamp = true; } + return Error::kNoError; } diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index ebd812470a..63029924f0 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -7,34 +7,28 @@ #include "bare_bones/algorithm.h" #include "bare_bones/vector.h" #include "bare_bones/xxhash.h" -#include "encoding.h" -#include "marked.h" #include "parser.h" #include "prometheus/hashdex.h" #include "prometheus/metric.h" #include "prometheus/textparse/escape.h" #include "prometheus/value.h" +#include "primitives/sample.h" + namespace PromPP::WAL::hashdex::scraper { template class Scraper { public: [[nodiscard]] Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { - metric_buffer_.initialize(buffer.size() / 4); - metadata_buffer_.initialize(buffer.size() / 128); - labels_.reserve(255); - - default_timestamp_ = default_timestamp; - - auto& tokenizer = parser_.tokenizer(); - tokenizer.tokenize({buffer.data(), buffer.data() + buffer.size()}); + metric_buffer_.initialize(buffer.size() / 2); + metadata_buffer_.initialize(buffer.size() / 4); + parser_.tokenizer().tokenize({buffer.data(), buffer.data() + buffer.size()}); while (true) { - switch (tokenizer.next()) { + switch (parser_.tokenizer().next()) { case Token::kEOF: case Token::kEOFWord: { - metric_buffer_.add_padding(); return parser_.validate_parse_result(); } @@ -55,12 +49,14 @@ class Scraper { case Token::kMetricName: case Token::kBraceOpen: { - if (const auto error = parse_metric(); error != Error::kNoError) [[unlikely]] { + if (const auto error = MetricParser{parser_, metric_buffer_, metric_buffer_.add_metric(default_timestamp)}.parse(); error != Error::kNoError) + [[unlikely]] { + metric_buffer_.remove_item(); return error; } - if (tokenizer.token() == Token::kExemplar) { - tokenizer.consume_comment(); + if (parser_.tokenizer().token() == Token::kExemplar) { + parser_.tokenizer().consume_comment(); } break; @@ -77,11 +73,9 @@ class Scraper { public: explicit MetricsWrapper(const Scraper& scraper) : scraper_(scraper) {} - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t size() const noexcept { return scraper_.metric_buffer_.items_count(); } - [[nodiscard]] PROMPP_ALWAYS_INLINE auto begin() const noexcept { - return scraper_.metric_buffer_.begin(scraper_.parser_.tokenizer().buffer(), scraper_.default_timestamp()); - } - [[nodiscard]] PROMPP_ALWAYS_INLINE static auto end() noexcept { return MetricMarkupBuffer::end(); } + [[nodiscard]] PROMPP_LAMBDA_INLINE uint32_t size() const noexcept { return scraper_.metric_buffer_.items_count(); } + [[nodiscard]] PROMPP_LAMBDA_INLINE auto begin() const noexcept { return scraper_.metric_buffer_.begin(scraper_.parser_.tokenizer().buffer()); } + [[nodiscard]] PROMPP_LAMBDA_INLINE static auto end() noexcept { return MetricMarkupBuffer::end(); } private: const Scraper& scraper_; @@ -91,268 +85,434 @@ class Scraper { public: explicit MetadataWrapper(const Scraper& scraper) : scraper_(scraper) {} - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t size() const noexcept { return scraper_.metadata_buffer_.items_count(); } - [[nodiscard]] PROMPP_ALWAYS_INLINE auto begin() const noexcept { return scraper_.metadata_buffer_.begin(scraper_.parser_.tokenizer().buffer()); } - [[nodiscard]] PROMPP_ALWAYS_INLINE static auto end() noexcept { return MetadataMarkupBuffer::end(); } + [[nodiscard]] PROMPP_LAMBDA_INLINE uint32_t size() const noexcept { return scraper_.metadata_buffer_.items_count(); } + [[nodiscard]] PROMPP_LAMBDA_INLINE auto begin() const noexcept { return scraper_.metadata_buffer_.begin(scraper_.parser_.tokenizer().buffer()); } + [[nodiscard]] PROMPP_LAMBDA_INLINE static auto end() noexcept { return MetadataMarkupBuffer::end(); } private: const Scraper& scraper_; }; - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t size() const noexcept { return metric_buffer_.items_count(); } - [[nodiscard]] PROMPP_ALWAYS_INLINE auto begin() const noexcept { return metric_buffer_.begin(parser_.tokenizer().buffer(), default_timestamp_); } - [[nodiscard]] PROMPP_ALWAYS_INLINE static auto end() noexcept { return MetricMarkupBuffer::end(); } + [[nodiscard]] PROMPP_LAMBDA_INLINE uint32_t size() const noexcept { return metric_buffer_.items_count(); } + [[nodiscard]] PROMPP_LAMBDA_INLINE auto begin() const noexcept { return metric_buffer_.begin(parser_.tokenizer().buffer()); } + [[nodiscard]] PROMPP_LAMBDA_INLINE static auto end() noexcept { return MetricMarkupBuffer::end(); } [[nodiscard]] PROMPP_ALWAYS_INLINE MetricsWrapper metrics() const noexcept { return MetricsWrapper{*this}; } [[nodiscard]] PROMPP_ALWAYS_INLINE MetadataWrapper metadata() const noexcept { return MetadataWrapper{*this}; } - [[nodiscard]] PROMPP_ALWAYS_INLINE Primitives::Timestamp default_timestamp() const noexcept { return default_timestamp_; } - [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { - return metric_buffer_.allocated_memory() + metadata_buffer_.allocated_memory() + labels_.allocated_memory(); + return metric_buffer_.allocated_memory() + metadata_buffer_.allocated_memory(); } private: using Token = Prometheus::textparse::Token; - [[nodiscard]] Error parse_metadata() { - static constexpr auto get_metadata_type = [](Token token) PROMPP_LAMBDA_INLINE { - if (token == Token::kHelp) { - return Prometheus::MetadataType::kHelp; - } - if (token == Token::kType) { - return Prometheus::MetadataType::kType; - } +#pragma pack(push, 1) + struct MarkedString { + uint32_t offset{std::numeric_limits::max()}; + uint32_t length{std::numeric_limits::max()}; - return Prometheus::MetadataType::kUnit; - }; + [[nodiscard]] PROMPP_ALWAYS_INLINE static MarkedString create(const std::string_view& value, const std::string_view& buffer) noexcept { + return { + .offset = static_cast(value.data() - buffer.data()), + .length = static_cast(value.size()), + }; + } - auto& tokenizer = parser_.tokenizer(); - const auto type = tokenizer.token(); + [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_valid() const noexcept { + return offset != std::numeric_limits::max() && length != std::numeric_limits::max(); + } - if (tokenizer.next_non_whitespace() != Token::kMetricName) [[unlikely]] { - return Error::kUnexpectedToken; + [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_empty() const noexcept { return length == 0; } + + [[nodiscard]] std::string_view string_view(const std::string_view& buffer) const noexcept { + if (!is_valid()) [[unlikely]] { + return Prometheus::kMetricLabelName; + } + + return buffer.substr(offset, length); } + }; - auto metric_name = tokenizer.token_str(); - Prometheus::textparse::unquote(metric_name); + struct MarkedLabel { + MarkedString name{}; + MarkedString value; + }; - if (tokenizer.next_non_whitespace() != Token::kText) [[unlikely]] { - return Error::kUnexpectedToken; + struct MarkedLabelSet { + uint32_t count{}; + MarkedLabel labels[]; + + PROMPP_ALWAYS_INLINE void sort(const std::string_view& buffer) noexcept { + std::sort(labels, labels + count, + [&buffer](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { return a.name.string_view(buffer) < b.name.string_view(buffer); }); } - const auto text = tokenizer.token_str(); - if (const auto token = tokenizer.next_non_whitespace(); !BareBones::is_in(token, Token::kLinebreak, Token::kEOF)) [[unlikely]] { - return Error::kUnexpectedToken; + [[nodiscard]] PROMPP_ALWAYS_INLINE uint64_t hash(const std::string_view& buffer) const noexcept { + BareBones::XXHash3 hash; + for (uint32_t i = 0; i < count; ++i) { + const auto& [name, value] = labels[i]; + hash.extend(name.string_view(buffer), value.string_view(buffer)); + } + return hash.hash(); } + }; - if (type == Token::kHelp && !simdutf::validate_utf8(text.data(), text.size())) [[unlikely]] { - return Error::kInvalidUtf8; + struct MarkedMetric { + uint64_t hash{}; + Primitives::Sample sample{}; + MarkedLabelSet label_set; + + explicit MarkedMetric(Primitives::Timestamp timestamp) : sample(timestamp, 0.0) {} + + PROMPP_ALWAYS_INLINE void calculate_hash(const std::string_view& buffer) noexcept { + label_set.sort(buffer); + hash = label_set.hash(buffer); } - const auto buffer = tokenizer.buffer(); - metadata_buffer_.add(MarkedString::create(metric_name, buffer), MarkedString::create(text, buffer), get_metadata_type(type)); - return Error::kNoError; - } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t occupied_size() const noexcept { return sizeof(*this) + sizeof(MarkedLabel) * label_set.count; } + }; - [[nodiscard]] Error parse_metric() { - labels_.clear(); + struct MarkedMetadata { + MarkedString metric_name{}; + MarkedString text{}; + Prometheus::MetadataType type{}; - marked_sample_ = {}; - marked_sample_.sample.timestamp() = default_timestamp_; + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t occupied_size() const noexcept { return sizeof(*this); } + }; +#pragma pack(pop) - bool have_metric_name = false; - auto& tokenizer = parser_.tokenizer(); + public: + class Metric { + public: + using MarkedItem = MarkedMetric; - const uint32_t metric_offset = tokenizer.token_str().data() - tokenizer.buffer().data(); + explicit Metric(std::string_view buffer, const MarkedMetric* item) : buffer_(buffer), item_(item) {} - if (tokenizer.token() == Token::kMetricName) [[likely]] { - labels_.push_back(MarkedLabel{.value = MarkedString::create(tokenizer.token_str(), tokenizer.buffer())}); + [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetric* item() const noexcept { return item_; } + PROMPP_ALWAYS_INLINE void set_item(const MarkedMetric* item) noexcept { item_ = item; } - have_metric_name = true; - tokenizer.next_non_whitespace(); - } else if (tokenizer.token() == Token::kWhitespace) [[likely]] { - tokenizer.next(); - } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint64_t hash() const noexcept { return item_->hash; } - if (tokenizer.token() == Token::kBraceOpen) [[likely]] { - if (const auto error = tokenize_label_set(have_metric_name); error != Error::kNoError) { - return error; + template + void read(Timeseries& timeseries) const { + timeseries.label_set().reserve(item_->label_set.count); + for (uint32_t i = 0; i < item_->label_set.count; ++i) { + const auto& [name, value] = item_->label_set.labels[i]; + timeseries.label_set().append(name.string_view(buffer_), value.string_view(buffer_)); } - tokenizer.next_non_whitespace(); - } else if (!parser_.is_value_token()) [[unlikely]] { - return Error::kUnexpectedToken; + timeseries.samples().emplace_back(item_->sample); } - if (!have_metric_name) [[unlikely]] { - return Error::kNoMetricName; - } + private: + std::string_view buffer_; + const MarkedMetric* item_{}; + }; - const auto error = parse_metric_suffix(); + class Metadata { + public: + using MarkedItem = MarkedMetadata; - if (error == Error::kNoError) [[likely]] { - encode_metric_data(metric_offset); - } + explicit Metadata(std::string_view buffer, const MarkedMetadata* item) : buffer_(buffer), item_(item) {} - return error; - } + [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetadata* item() const noexcept { return item_; } + PROMPP_ALWAYS_INLINE void set_item(const MarkedMetadata* item) noexcept { item_ = item; } - [[nodiscard]] Error tokenize_label_set(bool& have_metric_name) noexcept { - auto& tokenizer = parser_.tokenizer(); - tokenizer.next_non_whitespace(); + [[nodiscard]] PROMPP_ALWAYS_INLINE Prometheus::MetadataType type() const noexcept { return item_->type; } + [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view metric_name() const noexcept { return item_->metric_name.string_view(buffer_); } + [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view text() const noexcept { return item_->text.string_view(buffer_); } - while (tokenizer.token() != Token::kBraceClose) { - MarkedLabel label; - if (const auto error = get_label_name(label.name); error != Error::kNoError) [[unlikely]] { - return error; + private: + std::string_view buffer_; + const MarkedMetadata* item_{}; + }; + + private: + template + class MarkupBuffer { + public: + class IteratorSentinel {}; + + class Iterator { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = Item; + using difference_type = ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + using MarkedItem = typename Item::MarkedItem; + + Iterator(std::string_view buffer, const MarkupBuffer* markup_buffer) + : item_(buffer, reinterpret_cast(markup_buffer->buffer().data())), items_count_(markup_buffer->items_count()) {} + + [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type& operator*() const noexcept { return item_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type* operator->() const noexcept { return &item_; } + + PROMPP_ALWAYS_INLINE Iterator& operator++() noexcept { + item_.set_item(reinterpret_cast(reinterpret_cast(item_.item()) + item_.item()->occupied_size())); + --items_count_; + return *this; } - if (tokenizer.next_non_whitespace() == Token::kEqual) [[likely]] { - if (tokenizer.next_non_whitespace() != Token::kLabelValue) [[unlikely]] { - return Error::kUnexpectedToken; - } + PROMPP_ALWAYS_INLINE Iterator operator++(int) noexcept { + const auto it = *this; + ++*this; + return it; + } - if (const auto error = get_quoted_value(label.value); error != Error::kNoError) [[unlikely]] { - return error; - } + PROMPP_ALWAYS_INLINE bool operator==(const IteratorSentinel&) const noexcept { return items_count_ == 0; } + + private: + Item item_; + uint32_t items_count_; + }; + + [[nodiscard]] PROMPP_ALWAYS_INLINE const BareBones::Vector& buffer() const noexcept { return buffer_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t items_count() const noexcept { return items_count_; } + + PROMPP_ALWAYS_INLINE void remove_item() noexcept { --items_count_; } + + PROMPP_ALWAYS_INLINE void initialize(size_t reserve) noexcept { + buffer_.clear(); + buffer_.reserve(reserve); + items_count_ = 0; + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer) const noexcept { return {buffer, this}; } + [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } - labels_.push_back(label); + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return buffer_.allocated_memory(); } + + protected: + BareBones::Vector buffer_; + uint32_t items_count_{}; + }; + + class MetricMarkupBuffer : public MarkupBuffer { + public: + PROMPP_ALWAYS_INLINE MarkedMetric* add_metric(Primitives::Timestamp default_timestamp) noexcept { + ++this->items_count_; + + const auto offset = this->buffer_.size(); + this->buffer_.resize(offset + sizeof(MarkedMetric)); + return new (reinterpret_cast(this->buffer_.data() + offset)) MarkedMetric(default_timestamp); + } + + PROMPP_ALWAYS_INLINE void add_label(const MarkedLabel& label, MarkedMetric*& metric) noexcept { + if (label.value.is_empty()) [[unlikely]] { + return; + } + + const auto offset = reinterpret_cast(metric) - this->buffer_.data(); + this->buffer_.push_back(reinterpret_cast(&label), reinterpret_cast(&label) + sizeof(label)); + metric = reinterpret_cast(this->buffer_.data() + offset); + ++metric->label_set.count; + } + }; + class MetadataMarkupBuffer : public MarkupBuffer { + public: + PROMPP_ALWAYS_INLINE void add(MarkedString metric_name, MarkedString text, Prometheus::MetadataType type) noexcept { + ++this->items_count_; + + const auto offset = this->buffer_.size(); + this->buffer_.resize(offset + sizeof(MarkedMetadata)); + new (reinterpret_cast(this->buffer_.data() + offset)) MarkedMetadata{ + .metric_name = metric_name, + .text = text, + .type = type, + }; + } + }; + + class MetricParser { + public: + MetricParser(Parser& parser, MetricMarkupBuffer& markup_buffer, MarkedMetric* metric) : parser_(parser), markup_buffer_(markup_buffer), metric_(metric) {} + + [[nodiscard]] Error parse() noexcept { + bool have_metric_name = false; + auto& tokenizer = parser_.tokenizer(); + if (tokenizer.token() == Token::kMetricName) [[likely]] { + markup_buffer_.add_label(MarkedLabel{.value = MarkedString::create(tokenizer.token_str(), tokenizer.buffer())}, metric_); + have_metric_name = true; + tokenizer.next_non_whitespace(); + } else if (tokenizer.token() == Token::kWhitespace) [[likely]] { tokenizer.next(); - } else { - if (!have_metric_name) [[unlikely]] { - labels_.push_back(MarkedLabel{.value = label.name}); + } - have_metric_name = true; - } else { - return Error::kUnexpectedToken; + if (tokenizer.token() == Token::kBraceOpen) [[likely]] { + if (const auto error = tokenize_label_set(have_metric_name); error != Error::kNoError) { + return error; } + + tokenizer.next_non_whitespace(); + } else if (!parser_.is_value_token()) [[unlikely]] { + return Error::kUnexpectedToken; } - if (tokenizer.token() != Token::kComma && tokenizer.token() != Token::kWhitespace) { - break; + if (!have_metric_name) [[unlikely]] { + return Error::kNoMetricName; } - tokenizer.next_non_whitespace(); + metric_->calculate_hash(tokenizer.buffer()); + return parse_metric_suffix(); } - return tokenizer.token() == Token::kBraceClose ? Error::kNoError : Error::kUnexpectedToken; - } + private: + Parser& parser_; + MetricMarkupBuffer& markup_buffer_; + MarkedMetric* metric_; - [[nodiscard]] Error get_label_name(MarkedString& label_name) const noexcept { - auto& tokenizer = parser_.tokenizer(); + [[nodiscard]] Error tokenize_label_set(bool& have_metric_name) noexcept { + auto& tokenizer = parser_.tokenizer(); + tokenizer.next_non_whitespace(); - if (tokenizer.token() == Token::kLabelName) [[likely]] { - label_name = MarkedString::create(tokenizer.token_str(), tokenizer.buffer()); - return Error::kNoError; - } - if (tokenizer.token() == Token::kQuotedString) { - return get_quoted_value(label_name); - } + while (tokenizer.token() != Token::kBraceClose) { + MarkedLabel label; + if (const auto error = get_label_name(label.name); error != Error::kNoError) [[unlikely]] { + return error; + } - return Error::kUnexpectedToken; - } + if (tokenizer.next_non_whitespace() == Token::kEqual) [[likely]] { + if (tokenizer.next_non_whitespace() != Token::kLabelValue) [[unlikely]] { + return Error::kUnexpectedToken; + } - [[nodiscard]] Error get_quoted_value(MarkedString& string) const noexcept { - auto& tokenizer = parser_.tokenizer(); + if (const auto error = get_quoted_value(label.value); error != Error::kNoError) [[unlikely]] { + return error; + } - auto value = tokenizer.token_str(); - Prometheus::textparse::unquote(value); + markup_buffer_.add_label(label, metric_); + tokenizer.next(); + } else { + if (!have_metric_name) [[unlikely]] { + markup_buffer_.add_label(MarkedLabel{.value = label.name}, metric_); + have_metric_name = true; + } else { + return Error::kUnexpectedToken; + } + } - auto copy_to = const_cast(value.data()); - Prometheus::textparse::unescape_label_value(value, [©_to](const std::string_view& piece_of_string) { - if (copy_to != piece_of_string.data()) [[unlikely]] { - memmove(copy_to, piece_of_string.data(), piece_of_string.size()); - } + if (tokenizer.token() != Token::kComma && tokenizer.token() != Token::kWhitespace) { + break; + } - copy_to += piece_of_string.size(); - }); - value.remove_suffix(value.size() - (copy_to - value.data())); + tokenizer.next_non_whitespace(); + } - if (!simdutf::validate_utf8(value.data(), value.size())) [[unlikely]] { - return Error::kInvalidUtf8; + return tokenizer.token() == Token::kBraceClose ? Error::kNoError : Error::kUnexpectedToken; } - string = MarkedString::create(value, tokenizer.buffer()); - return Error::kNoError; - } + [[nodiscard]] Error get_label_name(MarkedString& label_name) const noexcept { + auto& tokenizer = parser_.tokenizer(); + + if (tokenizer.token() == Token::kLabelName) [[likely]] { + label_name = MarkedString::create(tokenizer.token_str(), tokenizer.buffer()); + return Error::kNoError; + } + if (tokenizer.token() == Token::kQuotedString) { + return get_quoted_value(label_name); + } - [[nodiscard]] Error parse_metric_suffix() noexcept { - if (!parser_.is_value_token()) [[unlikely]] { return Error::kUnexpectedToken; } - if (const auto error = parse_sample(); error != Error::kNoError) { - return error; - } + [[nodiscard]] Error get_quoted_value(MarkedString& string) const noexcept { + auto& tokenizer = parser_.tokenizer(); - return parser_.validate_parse_sample_result(); - } + auto value = tokenizer.token_str(); + Prometheus::textparse::unquote(value); - [[nodiscard]] Error parse_sample() noexcept { - auto& tokenizer = parser_.tokenizer(); + auto copy_to = const_cast(value.data()); + Prometheus::textparse::unescape_label_value(value, [©_to](const std::string_view& piece_of_string) { + if (copy_to != piece_of_string.data()) [[unlikely]] { + memmove(copy_to, piece_of_string.data(), piece_of_string.size()); + } + + copy_to += piece_of_string.size(); + }); + value.remove_suffix(value.size() - (copy_to - value.data())); + + if (!simdutf::validate_utf8(value.data(), value.size())) [[unlikely]] { + return Error::kInvalidUtf8; + } - if (!parse_numeric_value(tokenizer.token_str(), marked_sample_.sample.value())) [[unlikely]] { - return Error::kInvalidValue; + string = MarkedString::create(value, tokenizer.buffer()); + return Error::kNoError; } - if (std::isnan(marked_sample_.sample.value())) [[unlikely]] { - marked_sample_.sample.value() = Prometheus::kNormalNan; + + [[nodiscard]] Error parse_metric_suffix() noexcept { + if (!parser_.is_value_token()) [[unlikely]] { + return Error::kUnexpectedToken; + } + + if (const auto error = parse_sample(); error != Error::kNoError) { + return error; + } + + return parser_.validate_parse_sample_result(); } - tokenizer.next_non_whitespace(); + [[nodiscard]] Error parse_sample() noexcept { + auto& tokenizer = parser_.tokenizer(); - return parser_.parse_timestamp(marked_sample_.sample.timestamp(), marked_sample_.has_ts); - } + if (!parse_numeric_value(tokenizer.token_str(), metric_->sample.value())) [[unlikely]] { + return Error::kInvalidValue; + } + if (std::isnan(metric_->sample.value())) [[unlikely]] { + metric_->sample.value() = Prometheus::kNormalNan; + } - void encode_metric_data(const uint32_t metric_offset) noexcept { - metric_buffer_.add_metric(metric_offset); + tokenizer.next_non_whitespace(); - const auto base_buffer = parser_.tokenizer().buffer(); - sort_and_filter_labels(base_buffer); + return parser_.parse_timestamp(metric_->sample.timestamp()); + } + }; - metric_buffer_.bytes_enlarge(encoding::metric_maximum_encoding_size(labels_.size())); + [[nodiscard]] Error parse_metadata() { + static constexpr auto get_metadata_type = [](Token token) PROMPP_LAMBDA_INLINE { + if (token == Token::kHelp) { + return Prometheus::MetadataType::kHelp; + } + if (token == Token::kType) { + return Prometheus::MetadataType::kType; + } - const encoding::LayoutMarker layout = - encoding::LayoutMarker::make(marked_sample_.has_ts, labels_.size(), encoding::SampleCodec::value_type(marked_sample_.sample.value())); - metric_buffer_.add_layout_and_count(layout, labels_.size()); + return Prometheus::MetadataType::kUnit; + }; - append_labels_hash_and_encode(metric_offset, base_buffer); - metric_buffer_.add_sample(layout, marked_sample_.sample); - } + auto& tokenizer = parser_.tokenizer(); + const auto type = tokenizer.token(); - void sort_and_filter_labels(std::string_view base_buffer) noexcept { - const auto it = std::remove_if(labels_.begin(), labels_.end(), [](const MarkedLabel& label) PROMPP_LAMBDA_INLINE { return label.value.is_empty(); }); - labels_.erase(it, labels_.end()); + if (tokenizer.next_non_whitespace() != Token::kMetricName) [[unlikely]] { + return Error::kUnexpectedToken; + } - const auto compare = [buffer = base_buffer](const MarkedLabel& a, const MarkedLabel& b) - PROMPP_LAMBDA_INLINE { return a.name.view(buffer) < b.name.view(buffer); }; + auto metric_name = tokenizer.token_str(); + Prometheus::textparse::unquote(metric_name); - std::sort(labels_.begin(), labels_.end(), compare); - } + if (tokenizer.next_non_whitespace() != Token::kText) [[unlikely]] { + return Error::kUnexpectedToken; + } - void append_labels_hash_and_encode(uint32_t metric_offset, std::string_view base_buffer) noexcept { - BareBones::XXHash3 hash; - for (const auto& source_label : labels_) { - hash.extend(source_label.name.view(base_buffer), source_label.value.view(base_buffer)); + const auto text = tokenizer.token_str(); + if (const auto token = tokenizer.next_non_whitespace(); !BareBones::is_in(token, Token::kLinebreak, Token::kEOF)) [[unlikely]] { + return Error::kUnexpectedToken; + } - MarkedLabel encoded_label = source_label; - if (!encoded_label.name.is_reserved_name()) [[likely]] { - encoded_label.name.offset -= metric_offset; - } - encoded_label.value.offset -= metric_offset; - metric_buffer_.add_label(encoded_label); + if (type == Token::kHelp && !simdutf::validate_utf8(text.data(), text.size())) [[unlikely]] { + return Error::kInvalidUtf8; } - metric_buffer_.add_hash(hash.hash()); + const auto buffer = tokenizer.buffer(); + metadata_buffer_.add(MarkedString::create(metric_name, buffer), MarkedString::create(text, buffer), get_metadata_type(type)); + return Error::kNoError; } Parser parser_; MetricMarkupBuffer metric_buffer_; MetadataMarkupBuffer metadata_buffer_; - BareBones::Vector labels_; - Primitives::Timestamp default_timestamp_{}; - MarkedSample marked_sample_{}; }; using PrometheusScraper = Scraper; From 72bd60de336e447094ed5e21a7b278b87c59b73c Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 13 May 2026 10:08:40 +0000 Subject: [PATCH 43/48] scraper 4656f7137 --- pp/wal/benchmarks/scraper_benchmark.cpp | 32 +-- pp/wal/hashdex/scraper/parser.h | 13 +- pp/wal/hashdex/scraper/scraper.h | 289 +++++++++++++++++++----- 3 files changed, 258 insertions(+), 76 deletions(-) diff --git a/pp/wal/benchmarks/scraper_benchmark.cpp b/pp/wal/benchmarks/scraper_benchmark.cpp index 1a3f09177e..765a87c72f 100644 --- a/pp/wal/benchmarks/scraper_benchmark.cpp +++ b/pp/wal/benchmarks/scraper_benchmark.cpp @@ -59,27 +59,27 @@ void ScraperParse(benchmark::State& state) { } } -void ScraperRead(benchmark::State& state) { - ZoneScoped; - auto str = get_file_content(); +// void ScraperRead(benchmark::State& state) { +// ZoneScoped; +// auto str = get_file_content(); - PrometheusScraper scraper; - std::ignore = scraper.parse(str, 0); +// PrometheusScraper scraper; +// std::ignore = scraper.parse(str, 0); - PromPP::Primitives::TimeseriesSemiview ts_buf; +// PromPP::Primitives::TimeseriesSemiview ts_buf; - for ([[maybe_unused]] auto _ : state) { - for (auto& metric : scraper.metrics()) { - if (metric.hash() % 2 == 0) { - ts_buf.clear(); - metric.read(ts_buf); - } - } - } -} +// for ([[maybe_unused]] auto _ : state) { +// for (auto& metric : scraper.metrics()) { +// if (metric.hash() % 2 == 0) { +// ts_buf.clear(); +// metric.read(ts_buf); +// } +// } +// } +// } BENCHMARK(Parser)->ComputeStatistics("min", benchmark::min_time); BENCHMARK(ScraperParse)->ComputeStatistics("min", benchmark::min_time); -BENCHMARK(ScraperRead)->ComputeStatistics("min", benchmark::min_time); +// BENCHMARK(ScraperRead)->ComputeStatistics("min", benchmark::min_time); } // namespace diff --git a/pp/wal/hashdex/scraper/parser.h b/pp/wal/hashdex/scraper/parser.h index c539cb5941..76e03bdde6 100644 --- a/pp/wal/hashdex/scraper/parser.h +++ b/pp/wal/hashdex/scraper/parser.h @@ -31,13 +31,13 @@ template } template -concept ParserInterface = requires(Parser& parser, const Parser& const_parser, Primitives::Timestamp& timestamp) { +concept ParserInterface = requires(Parser& parser, const Parser& const_parser, Primitives::Timestamp& timestamp, bool& has_timestamp) { { parser.tokenizer() }; { Prometheus::textparse::TokenizerInterface }; { Prometheus::textparse::TokenizerInterface }; { const_parser.is_value_token() } -> std::same_as; - { parser.parse_timestamp(timestamp) } -> std::same_as; + { parser.parse_timestamp(timestamp, has_timestamp) } -> std::same_as; { const_parser.validate_parse_result() } -> std::same_as; { const_parser.validate_parse_sample_result() } -> std::same_as; }; @@ -59,13 +59,15 @@ class PrometheusParser { return (tokenizer_.token() == Token::kLinebreak || tokenizer_.token() == Token::kEOF) ? Error::kNoError : Error::kUnexpectedToken; } - [[nodiscard]] Error parse_timestamp(Primitives::Timestamp& timestamp) noexcept { + [[nodiscard]] Error parse_timestamp(Primitives::Timestamp& timestamp, bool& has_timestamp) noexcept { + has_timestamp = false; if (is_timestamp_token()) { if (!parse_numeric_value(tokenizer_.token_str(), timestamp)) [[unlikely]] { return Error::kInvalidTimestamp; } tokenizer_.next_non_whitespace(); + has_timestamp = true; } return Error::kNoError; @@ -96,7 +98,8 @@ class OpenMetricsParser { return (tokenizer_.token() == Token::kLinebreak || tokenizer_.token() == Token::kExemplar) ? Error::kNoError : Error::kUnexpectedToken; } - [[nodiscard]] Error parse_timestamp(Primitives::Timestamp& timestamp) noexcept { + [[nodiscard]] Error parse_timestamp(Primitives::Timestamp& timestamp, bool& has_timestamp) noexcept { + has_timestamp = false; if (tokenizer_.token() == Token::kTimestamp) { if (double float_timestamp; parse_timestamp_as_float(float_timestamp)) [[likely]] { timestamp = static_cast(float_timestamp * 1000.0); @@ -105,8 +108,8 @@ class OpenMetricsParser { } tokenizer_.next_non_whitespace(); + has_timestamp = true; } - return Error::kNoError; } diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 63029924f0..39fc8f402c 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -2,6 +2,7 @@ #include +#include #include #include "bare_bones/algorithm.h" @@ -21,12 +22,13 @@ template class Scraper { public: [[nodiscard]] Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { - metric_buffer_.initialize(buffer.size() / 2); - metadata_buffer_.initialize(buffer.size() / 4); - parser_.tokenizer().tokenize({buffer.data(), buffer.data() + buffer.size()}); + labels_.reserve(255); + + auto& tokenizer = parser_.tokenizer(); + tokenizer.tokenize({buffer.data(), buffer.data() + buffer.size()}); while (true) { - switch (parser_.tokenizer().next()) { + switch (tokenizer.next()) { case Token::kEOF: case Token::kEOFWord: { return parser_.validate_parse_result(); @@ -49,14 +51,14 @@ class Scraper { case Token::kMetricName: case Token::kBraceOpen: { - if (const auto error = MetricParser{parser_, metric_buffer_, metric_buffer_.add_metric(default_timestamp)}.parse(); error != Error::kNoError) - [[unlikely]] { + if (const auto error = MetricParser{parser_, metric_buffer_, metric_buffer_.add_metric(), labels_, default_timestamp}.parse(); + error != Error::kNoError) [[unlikely]] { metric_buffer_.remove_item(); return error; } - if (parser_.tokenizer().token() == Token::kExemplar) { - parser_.tokenizer().consume_comment(); + if (tokenizer.token() == Token::kExemplar) { + tokenizer.consume_comment(); } break; @@ -119,14 +121,14 @@ class Scraper { }; } - [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_valid() const noexcept { - return offset != std::numeric_limits::max() && length != std::numeric_limits::max(); + [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_reserved_name() const noexcept { + return offset == std::numeric_limits::max() && length == std::numeric_limits::max(); } [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_empty() const noexcept { return length == 0; } - [[nodiscard]] std::string_view string_view(const std::string_view& buffer) const noexcept { - if (!is_valid()) [[unlikely]] { + [[nodiscard]] std::string_view view(const std::string_view& buffer) const noexcept { + if (is_reserved_name()) [[unlikely]] { return Prometheus::kMetricLabelName; } @@ -139,38 +141,20 @@ class Scraper { MarkedString value; }; - struct MarkedLabelSet { - uint32_t count{}; - MarkedLabel labels[]; - - PROMPP_ALWAYS_INLINE void sort(const std::string_view& buffer) noexcept { - std::sort(labels, labels + count, - [&buffer](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { return a.name.string_view(buffer) < b.name.string_view(buffer); }); - } - - [[nodiscard]] PROMPP_ALWAYS_INLINE uint64_t hash(const std::string_view& buffer) const noexcept { - BareBones::XXHash3 hash; - for (uint32_t i = 0; i < count; ++i) { - const auto& [name, value] = labels[i]; - hash.extend(name.string_view(buffer), value.string_view(buffer)); - } - return hash.hash(); - } + struct MarkedSample { + Primitives::Sample sample{}; + bool has_ts{}; }; struct MarkedMetric { uint64_t hash{}; - Primitives::Sample sample{}; - MarkedLabelSet label_set; - - explicit MarkedMetric(Primitives::Timestamp timestamp) : sample(timestamp, 0.0) {} + uint32_t offset{}; + uint32_t offset_extra{}; + uint8_t bytes[]; - PROMPP_ALWAYS_INLINE void calculate_hash(const std::string_view& buffer) noexcept { - label_set.sort(buffer); - hash = label_set.hash(buffer); - } + // explicit MarkedMetric(Primitives::Timestamp timestamp) : sample(timestamp, 0.0) {} - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t occupied_size() const noexcept { return sizeof(*this) + sizeof(MarkedLabel) * label_set.count; } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t occupied_size() const noexcept { return sizeof(*this) + sizeof(MarkedLabel) /*add bytes skip!!!*/; } }; struct MarkedMetadata { @@ -199,7 +183,7 @@ class Scraper { timeseries.label_set().reserve(item_->label_set.count); for (uint32_t i = 0; i < item_->label_set.count; ++i) { const auto& [name, value] = item_->label_set.labels[i]; - timeseries.label_set().append(name.string_view(buffer_), value.string_view(buffer_)); + timeseries.label_set().append(name.view(buffer_), value.view(buffer_)); } timeseries.samples().emplace_back(item_->sample); @@ -220,8 +204,8 @@ class Scraper { PROMPP_ALWAYS_INLINE void set_item(const MarkedMetadata* item) noexcept { item_ = item; } [[nodiscard]] PROMPP_ALWAYS_INLINE Prometheus::MetadataType type() const noexcept { return item_->type; } - [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view metric_name() const noexcept { return item_->metric_name.string_view(buffer_); } - [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view text() const noexcept { return item_->text.string_view(buffer_); } + [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view metric_name() const noexcept { return item_->metric_name.view(buffer_); } + [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view text() const noexcept { return item_->text.view(buffer_); } private: std::string_view buffer_; @@ -282,7 +266,7 @@ class Scraper { [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer) const noexcept { return {buffer, this}; } [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } - [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return buffer_.allocated_memory(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return buffer_.size(); } protected: BareBones::Vector buffer_; @@ -291,12 +275,52 @@ class Scraper { class MetricMarkupBuffer : public MarkupBuffer { public: - PROMPP_ALWAYS_INLINE MarkedMetric* add_metric(Primitives::Timestamp default_timestamp) noexcept { + PROMPP_ALWAYS_INLINE MarkedMetric* add_metric() noexcept { ++this->items_count_; const auto offset = this->buffer_.size(); this->buffer_.resize(offset + sizeof(MarkedMetric)); - return new (reinterpret_cast(this->buffer_.data() + offset)) MarkedMetric(default_timestamp); + return std::construct_at(reinterpret_cast(this->buffer_.data() + offset)); + } + + PROMPP_ALWAYS_INLINE void add_count(uint32_t count, MarkedMetric*& metric) noexcept { + constexpr uint32_t kVarint1Byte = 0x80; // 2^7 + constexpr uint32_t kVarint2Byte = 0x4000; // 2^14 + constexpr uint32_t kVarint3Byte = 0x200000; // 2^21 + constexpr uint32_t kVarint4Byte = 0x10000000; // 2^28 + constexpr uint8_t kContinueBit = 0x80; + constexpr uint8_t kValueMask = 0x7F; + + const auto offset = reinterpret_cast(metric) - this->buffer_.data(); + + std::array tmp{}; + char* out = tmp.data(); + + if (count < kVarint1Byte) { + *out++ = static_cast(count); + } else if (count < kVarint2Byte) { + *out++ = static_cast((count & kValueMask) | kContinueBit); + *out++ = static_cast(count >> 7); + } else if (count < kVarint3Byte) { + *out++ = static_cast((count & kValueMask) | kContinueBit); + *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); + *out++ = static_cast(count >> 14); + } else if (count < kVarint4Byte) { + *out++ = static_cast((count & kValueMask) | kContinueBit); + *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); + *out++ = static_cast(((count >> 14) & kValueMask) | kContinueBit); + *out++ = static_cast(count >> 21); + } else { + *out++ = static_cast((count & kValueMask) | kContinueBit); + *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); + *out++ = static_cast(((count >> 14) & kValueMask) | kContinueBit); + *out++ = static_cast(((count >> 21) & kValueMask) | kContinueBit); + *out++ = static_cast(count >> 28); + } + + this->buffer_.push_back(tmp.data(), out); + + metric = reinterpret_cast(this->buffer_.data() + offset); } PROMPP_ALWAYS_INLINE void add_label(const MarkedLabel& label, MarkedMetric*& metric) noexcept { @@ -304,10 +328,116 @@ class Scraper { return; } + auto name = label.name; + if (name.is_reserved_name()) [[unlikely]] { + name.offset = 0; + name.length = 0; + } + + const uint8_t sz0 = encode_size(name.offset); + const uint8_t sz1 = encode_size(name.length); + const uint8_t sz2 = encode_size(label.value.offset); + const uint8_t sz3 = encode_size(label.value.length); + + const uint8_t layout = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); + + std::array tmp{}; + char* out = tmp.data(); + + *out++ = static_cast(layout); + + *reinterpret_cast(out) = name.offset; + out += sz0 + 1; + + *reinterpret_cast(out) = name.length; + out += sz1 + 1; + + *reinterpret_cast(out) = label.value.offset; + out += sz2 + 1; + + *reinterpret_cast(out) = label.value.length; + out += sz3 + 1; + const auto offset = reinterpret_cast(metric) - this->buffer_.data(); - this->buffer_.push_back(reinterpret_cast(&label), reinterpret_cast(&label) + sizeof(label)); + + this->buffer_.push_back(tmp.data(), out); + metric = reinterpret_cast(this->buffer_.data() + offset); - ++metric->label_set.count; + } + + PROMPP_ALWAYS_INLINE void add_sample(const MarkedSample& sample, MarkedMetric*& metric) noexcept { + const auto offset = reinterpret_cast(metric) - this->buffer_.data(); + + add_sample_internal(sample); + + metric = reinterpret_cast(this->buffer_.data() + offset); + } + + PROMPP_ALWAYS_INLINE void add_sample_internal(const MarkedSample& sample) noexcept { + uint8_t marker = sample.has_ts ? 0b10000000 : 0; + const double val = sample.sample.value(); + + auto flush = [&](uint8_t m) PROMPP_LAMBDA_INLINE { + this->buffer_.push_back(marker | m); + if (sample.has_ts) { + append(sample.sample.timestamp()); + } + }; + + if (std::isnan(val)) [[unlikely]] { + flush(0b00000100); // staleNaN + return; + } + + if (val == 0.0) [[unlikely]] { + flush(0b00000000); // zero + return; + } + + if (std::trunc(val) == val && val > 0.0) [[likely]] { // u integer + auto ival = static_cast(val); + if (ival <= std::numeric_limits::max()) [[unlikely]] { + flush(0b00000001); + append(static_cast(ival)); + return; + } + if (ival <= std::numeric_limits::max()) [[unlikely]] { + flush(0b00000010); + append(static_cast(ival)); + return; + } + if (ival <= std::numeric_limits::max()) [[likely]] { + flush(0b00000011); + append(static_cast(ival)); + return; + } + } + + float f = static_cast(val); + if (static_cast(f) == val) [[unlikely]] { + flush(0b00001000); // float32 + append(f); + } else { + flush(0b00001001); // double + append(val); + } + } + + private: + template + PROMPP_ALWAYS_INLINE void append(const T val) noexcept { + auto p = reinterpret_cast(&val); + this->buffer_.push_back(p, p + sizeof(T)); + } + + static uint8_t encode_size(uint32_t v) noexcept { + if (v <= 0xFF) + return 0; + if (v <= 0xFFFF) + return 1; + if (v <= 0xFFFFFF) + return 2; + return 3; } }; @@ -328,13 +458,29 @@ class Scraper { class MetricParser { public: - MetricParser(Parser& parser, MetricMarkupBuffer& markup_buffer, MarkedMetric* metric) : parser_(parser), markup_buffer_(markup_buffer), metric_(metric) {} + MetricParser(Parser& parser, + MetricMarkupBuffer& markup_buffer, + MarkedMetric* metric, + BareBones::Vector& labels, + Primitives::Timestamp timestamp) + : parser_(parser), markup_buffer_(markup_buffer), metric_(metric), labels_(labels), sample_{.sample = {timestamp, 0.0}} {} [[nodiscard]] Error parse() noexcept { + labels_.clear(); + bool have_metric_name = false; auto& tokenizer = parser_.tokenizer(); + + metric_->offset = tokenizer.token_str().data() - tokenizer.buffer().data(); + if (tokenizer.token() == Token::kMetricName) [[likely]] { - markup_buffer_.add_label(MarkedLabel{.value = MarkedString::create(tokenizer.token_str(), tokenizer.buffer())}, metric_); + // markup_buffer_.add_label(MarkedLabel{.value = MarkedString::create(tokenizer.token_str(), tokenizer.buffer())}, metric_); + + auto string = MarkedString::create(tokenizer.token_str(), tokenizer.buffer()); + string.offset -= metric_->offset; + auto label = MarkedLabel{.value = string}; + labels_.push_back(label); + have_metric_name = true; tokenizer.next_non_whitespace(); } else if (tokenizer.token() == Token::kWhitespace) [[likely]] { @@ -355,7 +501,29 @@ class Scraper { return Error::kNoMetricName; } - metric_->calculate_hash(tokenizer.buffer()); + // metric_->calculate_hash(tokenizer.buffer()); + // sort + std::sort(labels_.begin(), labels_.end(), [buffer = tokenizer.buffer()](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { + return a.name.view(buffer) < b.name.view(buffer); + }); + + // hash + { + BareBones::XXHash3 hash; + for (const auto& label : labels_) { + hash.extend(label.name.view(tokenizer.buffer()), label.value.view(tokenizer.buffer())); + } + metric_->hash = hash.hash(); + } + + // encode count + { markup_buffer_.add_count(labels_.size(), metric_); } + + // encode labels + for (const auto& label : labels_) { + markup_buffer_.add_label(label, metric_); + } + return parse_metric_suffix(); } @@ -363,6 +531,8 @@ class Scraper { Parser& parser_; MetricMarkupBuffer& markup_buffer_; MarkedMetric* metric_; + BareBones::Vector& labels_; + MarkedSample sample_; [[nodiscard]] Error tokenize_label_set(bool& have_metric_name) noexcept { auto& tokenizer = parser_.tokenizer(); @@ -383,11 +553,15 @@ class Scraper { return error; } - markup_buffer_.add_label(label, metric_); + // markup_buffer_.add_label(label, metric_); + labels_.push_back(label); + tokenizer.next(); } else { if (!have_metric_name) [[unlikely]] { - markup_buffer_.add_label(MarkedLabel{.value = label.name}, metric_); + // markup_buffer_.add_label(MarkedLabel{.value = label.name}, metric_); + labels_.push_back(label); + have_metric_name = true; } else { return Error::kUnexpectedToken; @@ -409,6 +583,7 @@ class Scraper { if (tokenizer.token() == Token::kLabelName) [[likely]] { label_name = MarkedString::create(tokenizer.token_str(), tokenizer.buffer()); + label_name.offset -= metric_->offset; return Error::kNoError; } if (tokenizer.token() == Token::kQuotedString) { @@ -439,6 +614,7 @@ class Scraper { } string = MarkedString::create(value, tokenizer.buffer()); + string.offset -= metric_->offset; return Error::kNoError; } @@ -451,22 +627,24 @@ class Scraper { return error; } + markup_buffer_.add_sample(sample_, metric_); + return parser_.validate_parse_sample_result(); } [[nodiscard]] Error parse_sample() noexcept { auto& tokenizer = parser_.tokenizer(); - if (!parse_numeric_value(tokenizer.token_str(), metric_->sample.value())) [[unlikely]] { + if (!parse_numeric_value(tokenizer.token_str(), sample_.sample.value())) [[unlikely]] { return Error::kInvalidValue; } - if (std::isnan(metric_->sample.value())) [[unlikely]] { - metric_->sample.value() = Prometheus::kNormalNan; + if (std::isnan(sample_.sample.value())) [[unlikely]] { + sample_.sample.value() = Prometheus::kNormalNan; } tokenizer.next_non_whitespace(); - return parser_.parse_timestamp(metric_->sample.timestamp()); + return parser_.parse_timestamp(sample_.sample.timestamp(), sample_.has_ts); } }; @@ -513,6 +691,7 @@ class Scraper { Parser parser_; MetricMarkupBuffer metric_buffer_; MetadataMarkupBuffer metadata_buffer_; + BareBones::Vector labels_; }; using PrometheusScraper = Scraper; From 688d62ad643e0a118281b16e5a2890fa3cf2f7c0 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 13 May 2026 10:12:36 +0000 Subject: [PATCH 44/48] scraper 0027219f0 --- pp/wal/benchmarks/scraper_benchmark.cpp | 32 +-- pp/wal/hashdex/scraper/scraper.h | 327 ++++++++++++++++++------ 2 files changed, 261 insertions(+), 98 deletions(-) diff --git a/pp/wal/benchmarks/scraper_benchmark.cpp b/pp/wal/benchmarks/scraper_benchmark.cpp index 765a87c72f..1a3f09177e 100644 --- a/pp/wal/benchmarks/scraper_benchmark.cpp +++ b/pp/wal/benchmarks/scraper_benchmark.cpp @@ -59,27 +59,27 @@ void ScraperParse(benchmark::State& state) { } } -// void ScraperRead(benchmark::State& state) { -// ZoneScoped; -// auto str = get_file_content(); +void ScraperRead(benchmark::State& state) { + ZoneScoped; + auto str = get_file_content(); -// PrometheusScraper scraper; -// std::ignore = scraper.parse(str, 0); + PrometheusScraper scraper; + std::ignore = scraper.parse(str, 0); -// PromPP::Primitives::TimeseriesSemiview ts_buf; + PromPP::Primitives::TimeseriesSemiview ts_buf; -// for ([[maybe_unused]] auto _ : state) { -// for (auto& metric : scraper.metrics()) { -// if (metric.hash() % 2 == 0) { -// ts_buf.clear(); -// metric.read(ts_buf); -// } -// } -// } -// } + for ([[maybe_unused]] auto _ : state) { + for (auto& metric : scraper.metrics()) { + if (metric.hash() % 2 == 0) { + ts_buf.clear(); + metric.read(ts_buf); + } + } + } +} BENCHMARK(Parser)->ComputeStatistics("min", benchmark::min_time); BENCHMARK(ScraperParse)->ComputeStatistics("min", benchmark::min_time); -// BENCHMARK(ScraperRead)->ComputeStatistics("min", benchmark::min_time); +BENCHMARK(ScraperRead)->ComputeStatistics("min", benchmark::min_time); } // namespace diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 39fc8f402c..88c4fa31c1 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -23,6 +23,7 @@ class Scraper { public: [[nodiscard]] Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { labels_.reserve(255); + default_timestamp_ = default_timestamp; auto& tokenizer = parser_.tokenizer(); tokenizer.tokenize({buffer.data(), buffer.data() + buffer.size()}); @@ -51,7 +52,9 @@ class Scraper { case Token::kMetricName: case Token::kBraceOpen: { - if (const auto error = MetricParser{parser_, metric_buffer_, metric_buffer_.add_metric(), labels_, default_timestamp}.parse(); + if (const auto error = MetricParser{parser_, metric_buffer_, labels_, static_cast(tokenizer.token_str().data() - tokenizer.buffer().data()), + default_timestamp} + .parse(); error != Error::kNoError) [[unlikely]] { metric_buffer_.remove_item(); return error; @@ -76,7 +79,9 @@ class Scraper { explicit MetricsWrapper(const Scraper& scraper) : scraper_(scraper) {} [[nodiscard]] PROMPP_LAMBDA_INLINE uint32_t size() const noexcept { return scraper_.metric_buffer_.items_count(); } - [[nodiscard]] PROMPP_LAMBDA_INLINE auto begin() const noexcept { return scraper_.metric_buffer_.begin(scraper_.parser_.tokenizer().buffer()); } + [[nodiscard]] PROMPP_LAMBDA_INLINE auto begin() const noexcept { + return scraper_.metric_buffer_.begin(scraper_.parser_.tokenizer().buffer(), scraper_.default_timestamp()); + } [[nodiscard]] PROMPP_LAMBDA_INLINE static auto end() noexcept { return MetricMarkupBuffer::end(); } private: @@ -96,12 +101,14 @@ class Scraper { }; [[nodiscard]] PROMPP_LAMBDA_INLINE uint32_t size() const noexcept { return metric_buffer_.items_count(); } - [[nodiscard]] PROMPP_LAMBDA_INLINE auto begin() const noexcept { return metric_buffer_.begin(parser_.tokenizer().buffer()); } + [[nodiscard]] PROMPP_LAMBDA_INLINE auto begin() const noexcept { return metric_buffer_.begin(parser_.tokenizer().buffer(), default_timestamp_); } [[nodiscard]] PROMPP_LAMBDA_INLINE static auto end() noexcept { return MetricMarkupBuffer::end(); } [[nodiscard]] PROMPP_ALWAYS_INLINE MetricsWrapper metrics() const noexcept { return MetricsWrapper{*this}; } [[nodiscard]] PROMPP_ALWAYS_INLINE MetadataWrapper metadata() const noexcept { return MetadataWrapper{*this}; } + [[nodiscard]] PROMPP_ALWAYS_INLINE Primitives::Timestamp default_timestamp() const noexcept { return default_timestamp_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return metric_buffer_.allocated_memory() + metadata_buffer_.allocated_memory(); } @@ -148,21 +155,14 @@ class Scraper { struct MarkedMetric { uint64_t hash{}; - uint32_t offset{}; - uint32_t offset_extra{}; - uint8_t bytes[]; - - // explicit MarkedMetric(Primitives::Timestamp timestamp) : sample(timestamp, 0.0) {} - - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t occupied_size() const noexcept { return sizeof(*this) + sizeof(MarkedLabel) /*add bytes skip!!!*/; } + uint32_t base_offset{}; + uint32_t data_offset{}; }; struct MarkedMetadata { MarkedString metric_name{}; MarkedString text{}; Prometheus::MetadataType type{}; - - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t occupied_size() const noexcept { return sizeof(*this); } }; #pragma pack(pop) @@ -171,7 +171,8 @@ class Scraper { public: using MarkedItem = MarkedMetric; - explicit Metric(std::string_view buffer, const MarkedMetric* item) : buffer_(buffer), item_(item) {} + Metric(std::string_view buffer, const BareBones::Vector& bytes_buffer, const MarkedMetric* item, Primitives::Timestamp default_timestamp) + : buffer_(buffer), bytes_buffer_(bytes_buffer), item_(item), default_timestamp_(default_timestamp) {} [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetric* item() const noexcept { return item_; } PROMPP_ALWAYS_INLINE void set_item(const MarkedMetric* item) noexcept { item_ = item; } @@ -179,19 +180,141 @@ class Scraper { [[nodiscard]] PROMPP_ALWAYS_INLINE uint64_t hash() const noexcept { return item_->hash; } template - void read(Timeseries& timeseries) const { - timeseries.label_set().reserve(item_->label_set.count); - for (uint32_t i = 0; i < item_->label_set.count; ++i) { - const auto& [name, value] = item_->label_set.labels[i]; - timeseries.label_set().append(name.view(buffer_), value.view(buffer_)); + void read(Timeseries& ts) const { + const char* ptr = bytes_buffer_.data() + item_->data_offset; + const char* base = buffer_.data() + item_->base_offset; + + // decode label count + uint32_t labels_count = decode_varint(ptr); + ts.label_set().reserve(labels_count); + + // decode label + for (uint32_t i = 0; i < labels_count; ++i) { + const uint8_t layout = static_cast(*ptr++); + const uint8_t sz0 = (layout >> 0) & 0x3; + const uint8_t sz1 = (layout >> 2) & 0x3; + const uint8_t sz2 = (layout >> 4) & 0x3; + const uint8_t sz3 = (layout >> 6) & 0x3; + + auto read_val = [&](uint8_t sz) PROMPP_LAMBDA_INLINE -> uint32_t { + uint32_t v = 0; + memcpy(&v, ptr, sz + 1); + ptr += sz + 1; + return v; + }; + + uint32_t name_off = read_val(sz0); + uint32_t name_len = read_val(sz1); + uint32_t value_off = read_val(sz2); + uint32_t value_len = read_val(sz3); + + if (name_off == 0 && name_len == 0) [[unlikely]] { + ts.label_set().append(Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); + } else { + ts.label_set().append(std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); + } + } + + // decode sample + uint8_t marker = static_cast(*ptr++); + bool has_ts = (marker & 0b10000000) != 0; + uint8_t type = marker & 0b01111111; + + Primitives::Sample sample{}; + sample.timestamp() = default_timestamp_; + + if (has_ts) [[unlikely]] { + int64_t ts_val; + memcpy(&ts_val, ptr, sizeof(ts_val)); + ptr += sizeof(ts_val); + sample.timestamp() = ts_val; } - timeseries.samples().emplace_back(item_->sample); + double val; + + switch (type) { + case 0b00000011: { // uint32 + uint32_t tmp; + memcpy(&tmp, ptr, sizeof(tmp)); + ptr += sizeof(tmp); + val = static_cast(tmp); + break; + } + case 0b00000000: // zero + val = 0.0; + break; + case 0b00000001: { // uint8 + uint8_t tmp = static_cast(*ptr++); + val = static_cast(tmp); + break; + } + case 0b00000010: { // uint16 + uint16_t tmp; + memcpy(&tmp, ptr, sizeof(tmp)); + ptr += sizeof(tmp); + val = static_cast(tmp); + break; + } + case 0b00000100: // NaN + val = Prometheus::kNormalNan; + break; + case 0b00001000: { // float32 + float tmp; + memcpy(&tmp, ptr, sizeof(tmp)); + ptr += sizeof(tmp); + val = static_cast(tmp); + break; + } + case 0b00001001: { // double + double tmp; + memcpy(&tmp, ptr, sizeof(tmp)); + ptr += sizeof(tmp); + val = tmp; + break; + } + default: + val = Prometheus::kStaleNan; + break; + } + + sample.value() = val; + ts.samples().emplace_back(sample); } private: std::string_view buffer_; + const BareBones::Vector& bytes_buffer_; const MarkedMetric* item_{}; + Primitives::Timestamp default_timestamp_; + + static uint32_t decode_varint(const char*& ptr) noexcept { + uint32_t b0 = static_cast(*ptr++); + if ((b0 & 0x80) == 0) [[likely]] { + return b0; + } + + uint32_t b1 = static_cast(*ptr++); + uint32_t v = (b0 & 0x7F) | ((b1 & 0x7F) << 7); + if ((b1 & 0x80) == 0) { + return v; + } + + uint32_t b2 = static_cast(*ptr++); + v |= (b2 & 0x7F) << 14; + if ((b2 & 0x80) == 0) { + return v; + } + + uint32_t b3 = static_cast(*ptr++); + v |= (b3 & 0x7F) << 21; + if ((b3 & 0x80) == 0) { + return v; + } + + uint32_t b4 = static_cast(*ptr++); + v |= (b4 & 0x0F) << 28; + return v; + } }; class Metadata { @@ -234,7 +357,7 @@ class Scraper { [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type* operator->() const noexcept { return &item_; } PROMPP_ALWAYS_INLINE Iterator& operator++() noexcept { - item_.set_item(reinterpret_cast(reinterpret_cast(item_.item()) + item_.item()->occupied_size())); + item_.set_item(reinterpret_cast(reinterpret_cast(item_.item()) + sizeof(*item_.item()))); --items_count_; return *this; } @@ -273,17 +396,75 @@ class Scraper { uint32_t items_count_{}; }; - class MetricMarkupBuffer : public MarkupBuffer { + class MetricMarkupBuffer { public: - PROMPP_ALWAYS_INLINE MarkedMetric* add_metric() noexcept { - ++this->items_count_; + class IteratorSentinel {}; - const auto offset = this->buffer_.size(); - this->buffer_.resize(offset + sizeof(MarkedMetric)); - return std::construct_at(reinterpret_cast(this->buffer_.data() + offset)); + class Iterator { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = Metric; + using difference_type = ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + Iterator(std::string_view buffer, + const BareBones::Vector& bytes_buffer, + const MarkedMetric* ptr, + uint32_t items_count, + Primitives::Timestamp default_timestamp) + : item_(buffer, bytes_buffer, ptr, default_timestamp), ptr_(ptr), buffer_(buffer), bytes_buffer_(bytes_buffer), items_count_(items_count) {} + + [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type& operator*() const noexcept { return item_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type* operator->() const noexcept { return &item_; } + + PROMPP_ALWAYS_INLINE Iterator& operator++() noexcept { + item_.set_item(++ptr_); + --items_count_; + return *this; + } + + PROMPP_ALWAYS_INLINE Iterator operator++(int) noexcept { + auto tmp = *this; + ++(*this); + return tmp; + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE bool operator==(const IteratorSentinel&) const noexcept { return items_count_ == 0; } + + private: + Metric item_; + const MarkedMetric* ptr_{}; + std::string_view buffer_; + const BareBones::Vector& bytes_buffer_; + uint32_t items_count_{}; + }; + + [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer, Primitives::Timestamp default_timestamp) const noexcept { + return {buffer, bytes_buffer_, metric_buffer_.data(), metric_buffer_.size(), default_timestamp}; } - PROMPP_ALWAYS_INLINE void add_count(uint32_t count, MarkedMetric*& metric) noexcept { + [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } + + [[nodiscard]] PROMPP_ALWAYS_INLINE const BareBones::Vector& metric_buffer() const noexcept { return metric_buffer_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE const BareBones::Vector& bytes_buffer() const noexcept { return bytes_buffer_; } + + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t items_count() const noexcept { return metric_buffer_.size(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t bytes_count() const noexcept { return bytes_buffer_.size(); } + + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return metric_buffer_.size() * sizeof(MarkedMetric) + bytes_buffer_.size(); } + + PROMPP_ALWAYS_INLINE void remove_item() noexcept { + bytes_buffer_.resize(metric_buffer_.back().data_offset); + metric_buffer_.resize(metric_buffer_.size() - 1); + } + + PROMPP_ALWAYS_INLINE void add_hash(uint64_t hash) noexcept { metric_buffer_.back().hash = hash; } + PROMPP_ALWAYS_INLINE void add_metric(uint32_t global_offset) noexcept { + metric_buffer_.push_back(MarkedMetric{.base_offset = global_offset, .data_offset = bytes_count()}); + } + + PROMPP_ALWAYS_INLINE void add_count(uint32_t count) noexcept { constexpr uint32_t kVarint1Byte = 0x80; // 2^7 constexpr uint32_t kVarint2Byte = 0x4000; // 2^14 constexpr uint32_t kVarint3Byte = 0x200000; // 2^21 @@ -291,8 +472,6 @@ class Scraper { constexpr uint8_t kContinueBit = 0x80; constexpr uint8_t kValueMask = 0x7F; - const auto offset = reinterpret_cast(metric) - this->buffer_.data(); - std::array tmp{}; char* out = tmp.data(); @@ -318,24 +497,22 @@ class Scraper { *out++ = static_cast(count >> 28); } - this->buffer_.push_back(tmp.data(), out); - - metric = reinterpret_cast(this->buffer_.data() + offset); + bytes_buffer_.push_back(tmp.data(), out); } - PROMPP_ALWAYS_INLINE void add_label(const MarkedLabel& label, MarkedMetric*& metric) noexcept { - if (label.value.is_empty()) [[unlikely]] { - return; - } + PROMPP_ALWAYS_INLINE void add_label(MarkedLabel label) noexcept { + const auto& metric = metric_buffer_.back(); - auto name = label.name; - if (name.is_reserved_name()) [[unlikely]] { - name.offset = 0; - name.length = 0; + if (label.name.is_reserved_name()) [[unlikely]] { + label.name.offset = 0; + label.name.length = 0; + } else { + label.name.offset -= metric.base_offset; } + label.value.offset -= metric.base_offset; - const uint8_t sz0 = encode_size(name.offset); - const uint8_t sz1 = encode_size(name.length); + const uint8_t sz0 = encode_size(label.name.offset); + const uint8_t sz1 = encode_size(label.name.length); const uint8_t sz2 = encode_size(label.value.offset); const uint8_t sz3 = encode_size(label.value.length); @@ -346,10 +523,10 @@ class Scraper { *out++ = static_cast(layout); - *reinterpret_cast(out) = name.offset; + *reinterpret_cast(out) = label.name.offset; out += sz0 + 1; - *reinterpret_cast(out) = name.length; + *reinterpret_cast(out) = label.name.length; out += sz1 + 1; *reinterpret_cast(out) = label.value.offset; @@ -358,34 +535,22 @@ class Scraper { *reinterpret_cast(out) = label.value.length; out += sz3 + 1; - const auto offset = reinterpret_cast(metric) - this->buffer_.data(); - - this->buffer_.push_back(tmp.data(), out); - - metric = reinterpret_cast(this->buffer_.data() + offset); + bytes_buffer_.push_back(tmp.data(), out); } - PROMPP_ALWAYS_INLINE void add_sample(const MarkedSample& sample, MarkedMetric*& metric) noexcept { - const auto offset = reinterpret_cast(metric) - this->buffer_.data(); - - add_sample_internal(sample); - - metric = reinterpret_cast(this->buffer_.data() + offset); - } - - PROMPP_ALWAYS_INLINE void add_sample_internal(const MarkedSample& sample) noexcept { + PROMPP_ALWAYS_INLINE void add_sample(const MarkedSample& sample) noexcept { uint8_t marker = sample.has_ts ? 0b10000000 : 0; const double val = sample.sample.value(); auto flush = [&](uint8_t m) PROMPP_LAMBDA_INLINE { - this->buffer_.push_back(marker | m); + bytes_buffer_.push_back(marker | m); if (sample.has_ts) { append(sample.sample.timestamp()); } }; if (std::isnan(val)) [[unlikely]] { - flush(0b00000100); // staleNaN + flush(0b00000100); // NaN return; } @@ -424,10 +589,13 @@ class Scraper { } private: + BareBones::Vector metric_buffer_; + BareBones::Vector bytes_buffer_; + template PROMPP_ALWAYS_INLINE void append(const T val) noexcept { auto p = reinterpret_cast(&val); - this->buffer_.push_back(p, p + sizeof(T)); + bytes_buffer_.push_back(p, p + sizeof(T)); } static uint8_t encode_size(uint32_t v) noexcept { @@ -460,10 +628,12 @@ class Scraper { public: MetricParser(Parser& parser, MetricMarkupBuffer& markup_buffer, - MarkedMetric* metric, BareBones::Vector& labels, + uint32_t global_offset, Primitives::Timestamp timestamp) - : parser_(parser), markup_buffer_(markup_buffer), metric_(metric), labels_(labels), sample_{.sample = {timestamp, 0.0}} {} + : parser_(parser), markup_buffer_(markup_buffer), labels_(labels), sample_{.sample = {timestamp, 0.0}} { + markup_buffer_.add_metric(global_offset); + } [[nodiscard]] Error parse() noexcept { labels_.clear(); @@ -471,15 +641,8 @@ class Scraper { bool have_metric_name = false; auto& tokenizer = parser_.tokenizer(); - metric_->offset = tokenizer.token_str().data() - tokenizer.buffer().data(); - if (tokenizer.token() == Token::kMetricName) [[likely]] { - // markup_buffer_.add_label(MarkedLabel{.value = MarkedString::create(tokenizer.token_str(), tokenizer.buffer())}, metric_); - - auto string = MarkedString::create(tokenizer.token_str(), tokenizer.buffer()); - string.offset -= metric_->offset; - auto label = MarkedLabel{.value = string}; - labels_.push_back(label); + labels_.push_back(MarkedLabel{.value = MarkedString::create(tokenizer.token_str(), tokenizer.buffer())}); have_metric_name = true; tokenizer.next_non_whitespace(); @@ -501,8 +664,12 @@ class Scraper { return Error::kNoMetricName; } - // metric_->calculate_hash(tokenizer.buffer()); // sort + { + const auto it = std::remove_if(labels_.begin(), labels_.end(), [](const MarkedLabel& label) { return label.value.is_empty(); }); + labels_.erase(it, labels_.end()); + } + std::sort(labels_.begin(), labels_.end(), [buffer = tokenizer.buffer()](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { return a.name.view(buffer) < b.name.view(buffer); }); @@ -513,15 +680,15 @@ class Scraper { for (const auto& label : labels_) { hash.extend(label.name.view(tokenizer.buffer()), label.value.view(tokenizer.buffer())); } - metric_->hash = hash.hash(); + markup_buffer_.add_hash(hash.hash()); } // encode count - { markup_buffer_.add_count(labels_.size(), metric_); } + { markup_buffer_.add_count(labels_.size()); } // encode labels for (const auto& label : labels_) { - markup_buffer_.add_label(label, metric_); + markup_buffer_.add_label(label); } return parse_metric_suffix(); @@ -530,7 +697,6 @@ class Scraper { private: Parser& parser_; MetricMarkupBuffer& markup_buffer_; - MarkedMetric* metric_; BareBones::Vector& labels_; MarkedSample sample_; @@ -553,14 +719,12 @@ class Scraper { return error; } - // markup_buffer_.add_label(label, metric_); labels_.push_back(label); tokenizer.next(); } else { if (!have_metric_name) [[unlikely]] { - // markup_buffer_.add_label(MarkedLabel{.value = label.name}, metric_); - labels_.push_back(label); + labels_.push_back(MarkedLabel{.value = label.name}); have_metric_name = true; } else { @@ -583,7 +747,6 @@ class Scraper { if (tokenizer.token() == Token::kLabelName) [[likely]] { label_name = MarkedString::create(tokenizer.token_str(), tokenizer.buffer()); - label_name.offset -= metric_->offset; return Error::kNoError; } if (tokenizer.token() == Token::kQuotedString) { @@ -614,7 +777,6 @@ class Scraper { } string = MarkedString::create(value, tokenizer.buffer()); - string.offset -= metric_->offset; return Error::kNoError; } @@ -627,7 +789,7 @@ class Scraper { return error; } - markup_buffer_.add_sample(sample_, metric_); + markup_buffer_.add_sample(sample_); return parser_.validate_parse_sample_result(); } @@ -692,6 +854,7 @@ class Scraper { MetricMarkupBuffer metric_buffer_; MetadataMarkupBuffer metadata_buffer_; BareBones::Vector labels_; + Primitives::Timestamp default_timestamp_{}; }; using PrometheusScraper = Scraper; From 04bcf6bb09dacce187f46daa15e52ae07344599a Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 13 May 2026 14:42:44 +0000 Subject: [PATCH 45/48] scraper 9b62e6ae --- pp/wal/hashdex/scraper/scraper.h | 270 ++++++++++++++++++------------- 1 file changed, 157 insertions(+), 113 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index 88c4fa31c1..cbf6b609d4 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -16,12 +16,16 @@ #include "primitives/sample.h" +// #include "tracy/Tracy.hpp" + namespace PromPP::WAL::hashdex::scraper { template class Scraper { public: [[nodiscard]] Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { + // ZoneScopedN("Scraper::parse"); + labels_.clear(); labels_.reserve(255); default_timestamp_ = default_timestamp; @@ -181,7 +185,8 @@ class Scraper { template void read(Timeseries& ts) const { - const char* ptr = bytes_buffer_.data() + item_->data_offset; + // ZoneScopedN("Metric::read()"); + const char* ptr = reinterpret_cast(bytes_buffer_.data() + item_->data_offset); const char* base = buffer_.data() + item_->base_offset; // decode label count @@ -196,19 +201,12 @@ class Scraper { const uint8_t sz2 = (layout >> 4) & 0x3; const uint8_t sz3 = (layout >> 6) & 0x3; - auto read_val = [&](uint8_t sz) PROMPP_LAMBDA_INLINE -> uint32_t { - uint32_t v = 0; - memcpy(&v, ptr, sz + 1); - ptr += sz + 1; - return v; - }; - - uint32_t name_off = read_val(sz0); - uint32_t name_len = read_val(sz1); - uint32_t value_off = read_val(sz2); - uint32_t value_len = read_val(sz3); + const uint32_t name_off = read_val_partial(ptr, sz0); + const uint32_t name_len = read_val_partial(ptr, sz1); + const uint32_t value_off = read_val_partial(ptr, sz2); + const uint32_t value_len = read_val_partial(ptr, sz3); - if (name_off == 0 && name_len == 0) [[unlikely]] { + if (name_len == 0 && name_off == 0) [[unlikely]] { ts.label_set().append(Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); } else { ts.label_set().append(std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); @@ -233,45 +231,50 @@ class Scraper { double val; switch (type) { - case 0b00000011: { // uint32 - uint32_t tmp; - memcpy(&tmp, ptr, sizeof(tmp)); - ptr += sizeof(tmp); - val = static_cast(tmp); - break; - } case 0b00000000: // zero val = 0.0; break; + case 0b00000001: { // uint8 - uint8_t tmp = static_cast(*ptr++); - val = static_cast(tmp); + val = static_cast(*ptr++); break; } + case 0b00000010: { // uint16 - uint16_t tmp; - memcpy(&tmp, ptr, sizeof(tmp)); - ptr += sizeof(tmp); - val = static_cast(tmp); + uint16_t x = static_cast(ptr[0]) | (static_cast(ptr[1]) << 8); + ptr += 2; + val = static_cast(x); break; } - case 0b00000100: // NaN + + case 0b00000011: { // uint32 + uint32_t x; + std::memcpy(&x, ptr, sizeof(x)); + ptr += 4; + val = static_cast(x); + break; + } + + case 0b00000100: // NaN (normal) val = Prometheus::kNormalNan; break; + case 0b00001000: { // float32 - float tmp; - memcpy(&tmp, ptr, sizeof(tmp)); - ptr += sizeof(tmp); - val = static_cast(tmp); + float x; + std::memcpy(&x, ptr, sizeof(x)); + ptr += sizeof(x); + val = static_cast(x); break; } + case 0b00001001: { // double - double tmp; - memcpy(&tmp, ptr, sizeof(tmp)); - ptr += sizeof(tmp); - val = tmp; + double x; + std::memcpy(&x, ptr, sizeof(x)); + ptr += sizeof(x); + val = x; break; } + default: val = Prometheus::kStaleNan; break; @@ -287,7 +290,7 @@ class Scraper { const MarkedMetric* item_{}; Primitives::Timestamp default_timestamp_; - static uint32_t decode_varint(const char*& ptr) noexcept { + PROMPP_ALWAYS_INLINE static uint32_t decode_varint(const char*& ptr) noexcept { uint32_t b0 = static_cast(*ptr++); if ((b0 & 0x80) == 0) [[likely]] { return b0; @@ -315,6 +318,33 @@ class Scraper { v |= (b4 & 0x0F) << 28; return v; } + + PROMPP_ALWAYS_INLINE + static uint32_t read_val_partial(const char*& p, uint8_t sz) noexcept { + if (sz == 0) [[likely]] { + const uint32_t v = static_cast(p[0]); + p += 1; + return v; + } + + if (sz == 1) { + const uint32_t v = static_cast(static_cast(p[0])) | (static_cast(static_cast(p[1])) << 8); + p += 2; + return v; + } + + if (sz == 2) { + const uint32_t v = static_cast(static_cast(p[0])) | (static_cast(static_cast(p[1])) << 8) | + (static_cast(static_cast(p[2])) << 16); + p += 3; + return v; + } + + uint32_t v; + std::memcpy(&v, p, 4); + p += 4; + return v; + } }; class Metadata { @@ -465,51 +495,54 @@ class Scraper { } PROMPP_ALWAYS_INLINE void add_count(uint32_t count) noexcept { - constexpr uint32_t kVarint1Byte = 0x80; // 2^7 - constexpr uint32_t kVarint2Byte = 0x4000; // 2^14 - constexpr uint32_t kVarint3Byte = 0x200000; // 2^21 - constexpr uint32_t kVarint4Byte = 0x10000000; // 2^28 constexpr uint8_t kContinueBit = 0x80; constexpr uint8_t kValueMask = 0x7F; - std::array tmp{}; - char* out = tmp.data(); + const uint32_t offset = bytes_count(); - if (count < kVarint1Byte) { - *out++ = static_cast(count); - } else if (count < kVarint2Byte) { + if (count < (1u << 7)) [[likely]] { + bytes_buffer_.resize(bytes_buffer_.size() + 1); + char* out = bytes_buffer_.data() + offset; + *out = static_cast(count); + } else if (count < (1u << 14)) { + bytes_buffer_.resize(bytes_buffer_.size() + 2); + char* out = bytes_buffer_.data() + offset; *out++ = static_cast((count & kValueMask) | kContinueBit); - *out++ = static_cast(count >> 7); - } else if (count < kVarint3Byte) { + *out = static_cast(count >> 7); + } else if (count < (1u << 21)) { + bytes_buffer_.resize(bytes_buffer_.size() + 3); + char* out = bytes_buffer_.data() + offset; *out++ = static_cast((count & kValueMask) | kContinueBit); *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); - *out++ = static_cast(count >> 14); - } else if (count < kVarint4Byte) { + *out = static_cast(count >> 14); + } else if (count < (1u << 28)) { + bytes_buffer_.resize(bytes_buffer_.size() + 4); + char* out = bytes_buffer_.data() + offset; *out++ = static_cast((count & kValueMask) | kContinueBit); *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); *out++ = static_cast(((count >> 14) & kValueMask) | kContinueBit); - *out++ = static_cast(count >> 21); + *out = static_cast(count >> 21); } else { + bytes_buffer_.resize(bytes_buffer_.size() + 5); + char* out = bytes_buffer_.data() + offset; *out++ = static_cast((count & kValueMask) | kContinueBit); *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); *out++ = static_cast(((count >> 14) & kValueMask) | kContinueBit); *out++ = static_cast(((count >> 21) & kValueMask) | kContinueBit); - *out++ = static_cast(count >> 28); + *out = static_cast(count >> 28); } - - bytes_buffer_.push_back(tmp.data(), out); } PROMPP_ALWAYS_INLINE void add_label(MarkedLabel label) noexcept { - const auto& metric = metric_buffer_.back(); + const auto base_offset = metric_buffer_.back().base_offset; if (label.name.is_reserved_name()) [[unlikely]] { label.name.offset = 0; label.name.length = 0; } else { - label.name.offset -= metric.base_offset; + label.name.offset -= base_offset; } - label.value.offset -= metric.base_offset; + label.value.offset -= base_offset; const uint8_t sz0 = encode_size(label.name.offset); const uint8_t sz1 = encode_size(label.name.length); @@ -518,73 +551,44 @@ class Scraper { const uint8_t layout = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); - std::array tmp{}; - char* out = tmp.data(); + const uint32_t bytes_needed = sizeof(layout) + (sz0 + sz1 + sz2 + sz3) + 4; + const uint32_t offset = bytes_count(); + bytes_buffer_.resize(bytes_buffer_.size() + 17); + char* out = bytes_buffer_.data() + offset; *out++ = static_cast(layout); - *reinterpret_cast(out) = label.name.offset; + std::memcpy(out, &label.name.offset, sz0 + 1); out += sz0 + 1; - - *reinterpret_cast(out) = label.name.length; + std::memcpy(out, &label.name.length, sz1 + 1); out += sz1 + 1; - - *reinterpret_cast(out) = label.value.offset; + std::memcpy(out, &label.value.offset, sz2 + 1); out += sz2 + 1; + std::memcpy(out, &label.value.length, sz3 + 1); - *reinterpret_cast(out) = label.value.length; - out += sz3 + 1; - - bytes_buffer_.push_back(tmp.data(), out); + bytes_buffer_.resize(offset + bytes_needed); } PROMPP_ALWAYS_INLINE void add_sample(const MarkedSample& sample) noexcept { - uint8_t marker = sample.has_ts ? 0b10000000 : 0; const double val = sample.sample.value(); - - auto flush = [&](uint8_t m) PROMPP_LAMBDA_INLINE { - bytes_buffer_.push_back(marker | m); - if (sample.has_ts) { - append(sample.sample.timestamp()); - } - }; + const bool has_ts = sample.has_ts; if (std::isnan(val)) [[unlikely]] { - flush(0b00000100); // NaN + append_sample_marker(0b00000100, has_ts); // NaN + append_timestamp_if_needed(sample); return; } if (val == 0.0) [[unlikely]] { - flush(0b00000000); // zero + append_sample_marker(0b00000000, has_ts); // zero + append_timestamp_if_needed(sample); return; } - if (std::trunc(val) == val && val > 0.0) [[likely]] { // u integer - auto ival = static_cast(val); - if (ival <= std::numeric_limits::max()) [[unlikely]] { - flush(0b00000001); - append(static_cast(ival)); - return; - } - if (ival <= std::numeric_limits::max()) [[unlikely]] { - flush(0b00000010); - append(static_cast(ival)); - return; - } - if (ival <= std::numeric_limits::max()) [[likely]] { - flush(0b00000011); - append(static_cast(ival)); - return; - } - } - - float f = static_cast(val); - if (static_cast(f) == val) [[unlikely]] { - flush(0b00001000); // float32 - append(f); + if (std::trunc(val) == val && val > 0.0) [[likely]] { + append_integer_sample(sample, val); } else { - flush(0b00001001); // double - append(val); + append_floating_sample(sample, val); } } @@ -592,20 +596,57 @@ class Scraper { BareBones::Vector metric_buffer_; BareBones::Vector bytes_buffer_; - template - PROMPP_ALWAYS_INLINE void append(const T val) noexcept { + PROMPP_ALWAYS_INLINE void append_sample_marker(uint8_t type, bool has_ts) noexcept { bytes_buffer_.push_back((has_ts ? 0b10000000 : 0) | type); } + + PROMPP_ALWAYS_INLINE void append_timestamp_if_needed(const MarkedSample& sample) noexcept { + if (sample.has_ts) { + append_value(sample.sample.timestamp()); + } + } + + PROMPP_ALWAYS_INLINE void append_integer_sample(const MarkedSample& sample, double val) noexcept { + const bool has_ts = sample.has_ts; + const uint64_t uval = static_cast(val); + if (uval <= std::numeric_limits::max()) { + append_sample_marker(0b00000001, has_ts); + append_timestamp_if_needed(sample); + append_value(static_cast(uval)); + } else if (uval <= std::numeric_limits::max()) { + append_sample_marker(0b00000010, has_ts); + append_timestamp_if_needed(sample); + append_value(static_cast(uval)); + } else if (uval <= std::numeric_limits::max()) { + append_sample_marker(0b00000011, has_ts); + append_timestamp_if_needed(sample); + append_value(static_cast(uval)); + } else { + append_floating_sample(sample, val); + } + } + + PROMPP_ALWAYS_INLINE void append_floating_sample(const MarkedSample& sample, double val) noexcept { + const bool has_ts = sample.has_ts; + float f = static_cast(val); + if (static_cast(f) == val) [[unlikely]] { + append_sample_marker(0b00001000, has_ts); // float32 + append_timestamp_if_needed(sample); + append_value(f); + } else { + append_sample_marker(0b00001001, has_ts); // double + append_timestamp_if_needed(sample); + append_value(val); + } + } + + template + PROMPP_ALWAYS_INLINE void append_value(T val) noexcept { auto p = reinterpret_cast(&val); bytes_buffer_.push_back(p, p + sizeof(T)); } - static uint8_t encode_size(uint32_t v) noexcept { - if (v <= 0xFF) - return 0; - if (v <= 0xFFFF) - return 1; - if (v <= 0xFFFFFF) - return 2; - return 3; + PROMPP_ALWAYS_INLINE static uint8_t encode_size(uint32_t v) noexcept { + const uint32_t msb = (v == 0 ? 0 : 31 - std::countl_zero(v)); + return msb >> 3; } }; @@ -636,6 +677,7 @@ class Scraper { } [[nodiscard]] Error parse() noexcept { + // ZoneScopedN("MetricParser::parse"); labels_.clear(); bool have_metric_name = false; @@ -684,7 +726,9 @@ class Scraper { } // encode count - { markup_buffer_.add_count(labels_.size()); } + { + markup_buffer_.add_count(labels_.size()); + } // encode labels for (const auto& label : labels_) { From 6bbe70598ee1566f0021382f6b7f0f384e434117 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 13 May 2026 10:17:10 +0000 Subject: [PATCH 46/48] scraper 2036d7ea --- pp/wal/hashdex/scraper/scraper.h | 599 +++++++++++++------------------ 1 file changed, 258 insertions(+), 341 deletions(-) diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index cbf6b609d4..db71bc8bf0 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -2,7 +2,6 @@ #include -#include #include #include "bare_bones/algorithm.h" @@ -23,9 +22,7 @@ namespace PromPP::WAL::hashdex::scraper { template class Scraper { public: - [[nodiscard]] Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { - // ZoneScopedN("Scraper::parse"); - labels_.clear(); + [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { labels_.reserve(255); default_timestamp_ = default_timestamp; @@ -56,10 +53,7 @@ class Scraper { case Token::kMetricName: case Token::kBraceOpen: { - if (const auto error = MetricParser{parser_, metric_buffer_, labels_, static_cast(tokenizer.token_str().data() - tokenizer.buffer().data()), - default_timestamp} - .parse(); - error != Error::kNoError) [[unlikely]] { + if (const auto error = parse_metric(); error != Error::kNoError) [[unlikely]] { metric_buffer_.remove_item(); return error; } @@ -173,10 +167,16 @@ class Scraper { public: class Metric { public: - using MarkedItem = MarkedMetric; + using MarkedT = MarkedMetric; + + struct Context { + std::string_view buffer; + const BareBones::Vector& bytes_buffer; + Primitives::Timestamp default_timestamp; + }; - Metric(std::string_view buffer, const BareBones::Vector& bytes_buffer, const MarkedMetric* item, Primitives::Timestamp default_timestamp) - : buffer_(buffer), bytes_buffer_(bytes_buffer), item_(item), default_timestamp_(default_timestamp) {} + Metric(const Context& ctx, const MarkedMetric* item) + : buffer_(ctx.buffer), bytes_buffer_(ctx.bytes_buffer), item_(item), default_timestamp_(ctx.default_timestamp) {} [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetric* item() const noexcept { return item_; } PROMPP_ALWAYS_INLINE void set_item(const MarkedMetric* item) noexcept { item_ = item; } @@ -185,7 +185,6 @@ class Scraper { template void read(Timeseries& ts) const { - // ZoneScopedN("Metric::read()"); const char* ptr = reinterpret_cast(bytes_buffer_.data() + item_->data_offset); const char* base = buffer_.data() + item_->base_offset; @@ -221,13 +220,6 @@ class Scraper { Primitives::Sample sample{}; sample.timestamp() = default_timestamp_; - if (has_ts) [[unlikely]] { - int64_t ts_val; - memcpy(&ts_val, ptr, sizeof(ts_val)); - ptr += sizeof(ts_val); - sample.timestamp() = ts_val; - } - double val; switch (type) { @@ -280,6 +272,12 @@ class Scraper { break; } + if (has_ts) [[unlikely]] { + Primitives::Timestamp ts_val; + memcpy(&ts_val, ptr, sizeof(ts_val)); + sample.timestamp() = ts_val; + } + sample.value() = val; ts.samples().emplace_back(sample); } @@ -349,9 +347,13 @@ class Scraper { class Metadata { public: - using MarkedItem = MarkedMetadata; + using MarkedT = MarkedMetadata; + + struct Context { + std::string_view buffer; + }; - explicit Metadata(std::string_view buffer, const MarkedMetadata* item) : buffer_(buffer), item_(item) {} + Metadata(const Context& ctx, const MarkedMetadata* item) : buffer_(ctx.buffer), item_(item) {} [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetadata* item() const noexcept { return item_; } PROMPP_ALWAYS_INLINE void set_item(const MarkedMetadata* item) noexcept { item_ = item; } @@ -365,133 +367,78 @@ class Scraper { const MarkedMetadata* item_{}; }; - private: - template + template class MarkupBuffer { public: + using MarkedT = typename T::MarkedT; + using Context = typename T::Context; + class IteratorSentinel {}; class Iterator { public: using iterator_category = std::forward_iterator_tag; - using value_type = Item; + using value_type = T; using difference_type = ptrdiff_t; using pointer = value_type*; using reference = value_type&; - using MarkedItem = typename Item::MarkedItem; - Iterator(std::string_view buffer, const MarkupBuffer* markup_buffer) - : item_(buffer, reinterpret_cast(markup_buffer->buffer().data())), items_count_(markup_buffer->items_count()) {} + Iterator(const Context& ctx, const MarkedT* ptr, uint32_t items_count) : item_(ctx, ptr), ptr_(ptr), items_count_(items_count), ctx_(ctx) {} [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type& operator*() const noexcept { return item_; } [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type* operator->() const noexcept { return &item_; } PROMPP_ALWAYS_INLINE Iterator& operator++() noexcept { - item_.set_item(reinterpret_cast(reinterpret_cast(item_.item()) + sizeof(*item_.item()))); + item_.set_item(++ptr_); --items_count_; return *this; } PROMPP_ALWAYS_INLINE Iterator operator++(int) noexcept { - const auto it = *this; - ++*this; - return it; + auto tmp = *this; + ++(*this); + return tmp; } PROMPP_ALWAYS_INLINE bool operator==(const IteratorSentinel&) const noexcept { return items_count_ == 0; } private: - Item item_; + T item_; + const MarkedT* ptr_; uint32_t items_count_; + Context ctx_; }; - [[nodiscard]] PROMPP_ALWAYS_INLINE const BareBones::Vector& buffer() const noexcept { return buffer_; } - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t items_count() const noexcept { return items_count_; } - - PROMPP_ALWAYS_INLINE void remove_item() noexcept { --items_count_; } - - PROMPP_ALWAYS_INLINE void initialize(size_t reserve) noexcept { - buffer_.clear(); - buffer_.reserve(reserve); - items_count_ = 0; - } - - [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer) const noexcept { return {buffer, this}; } - [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } - - [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return buffer_.size(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t items_count() const noexcept { return buffer_.size(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return buffer_.size() * sizeof(MarkedT); } protected: - BareBones::Vector buffer_; - uint32_t items_count_{}; + BareBones::Vector buffer_; }; - class MetricMarkupBuffer { + class MetricMarkupBuffer : public MarkupBuffer { public: - class IteratorSentinel {}; - - class Iterator { - public: - using iterator_category = std::forward_iterator_tag; - using value_type = Metric; - using difference_type = ptrdiff_t; - using pointer = value_type*; - using reference = value_type&; - - Iterator(std::string_view buffer, - const BareBones::Vector& bytes_buffer, - const MarkedMetric* ptr, - uint32_t items_count, - Primitives::Timestamp default_timestamp) - : item_(buffer, bytes_buffer, ptr, default_timestamp), ptr_(ptr), buffer_(buffer), bytes_buffer_(bytes_buffer), items_count_(items_count) {} - - [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type& operator*() const noexcept { return item_; } - [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type* operator->() const noexcept { return &item_; } - - PROMPP_ALWAYS_INLINE Iterator& operator++() noexcept { - item_.set_item(++ptr_); - --items_count_; - return *this; - } - - PROMPP_ALWAYS_INLINE Iterator operator++(int) noexcept { - auto tmp = *this; - ++(*this); - return tmp; - } - - [[nodiscard]] PROMPP_ALWAYS_INLINE bool operator==(const IteratorSentinel&) const noexcept { return items_count_ == 0; } + using Base = MarkupBuffer; + using Iterator = typename Base::Iterator; + using IteratorSentinel = typename Base::IteratorSentinel; - private: - Metric item_; - const MarkedMetric* ptr_{}; - std::string_view buffer_; - const BareBones::Vector& bytes_buffer_; - uint32_t items_count_{}; - }; - - [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer, Primitives::Timestamp default_timestamp) const noexcept { - return {buffer, bytes_buffer_, metric_buffer_.data(), metric_buffer_.size(), default_timestamp}; + [[nodiscard]] Iterator begin(std::string_view buffer, Primitives::Timestamp default_ts) const noexcept { + return {typename Base::Context{buffer, bytes_buffer_, default_ts}, this->buffer_.data(), this->items_count()}; } + [[nodiscard]] static IteratorSentinel end() noexcept { return {}; } - [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } - - [[nodiscard]] PROMPP_ALWAYS_INLINE const BareBones::Vector& metric_buffer() const noexcept { return metric_buffer_; } - [[nodiscard]] PROMPP_ALWAYS_INLINE const BareBones::Vector& bytes_buffer() const noexcept { return bytes_buffer_; } - - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t items_count() const noexcept { return metric_buffer_.size(); } [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t bytes_count() const noexcept { return bytes_buffer_.size(); } - [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return metric_buffer_.size() * sizeof(MarkedMetric) + bytes_buffer_.size(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return this->buffer_.size() * sizeof(MarkedMetric) + bytes_buffer_.size(); } PROMPP_ALWAYS_INLINE void remove_item() noexcept { - bytes_buffer_.resize(metric_buffer_.back().data_offset); - metric_buffer_.resize(metric_buffer_.size() - 1); + bytes_buffer_.resize(this->buffer_.back().data_offset); + this->buffer_.resize(this->buffer_.size() - 1); } - PROMPP_ALWAYS_INLINE void add_hash(uint64_t hash) noexcept { metric_buffer_.back().hash = hash; } + PROMPP_ALWAYS_INLINE void add_hash(uint64_t hash) noexcept { this->buffer_.back().hash = hash; } PROMPP_ALWAYS_INLINE void add_metric(uint32_t global_offset) noexcept { - metric_buffer_.push_back(MarkedMetric{.base_offset = global_offset, .data_offset = bytes_count()}); + this->buffer_.push_back(MarkedMetric{.base_offset = global_offset, .data_offset = bytes_count()}); } PROMPP_ALWAYS_INLINE void add_count(uint32_t count) noexcept { @@ -534,7 +481,7 @@ class Scraper { } PROMPP_ALWAYS_INLINE void add_label(MarkedLabel label) noexcept { - const auto base_offset = metric_buffer_.back().base_offset; + const auto base_offset = this->buffer_.back().base_offset; if (label.name.is_reserved_name()) [[unlikely]] { label.name.offset = 0; @@ -573,75 +520,57 @@ class Scraper { const double val = sample.sample.value(); const bool has_ts = sample.has_ts; + constexpr uint32_t max_sample_bytes = 1 + sizeof(sample.sample); // marker + value + timestamp + const uint32_t offset = bytes_count(); + bytes_buffer_.resize(offset + max_sample_bytes); + char* out = bytes_buffer_.data() + offset; + char* start = out; + if (std::isnan(val)) [[unlikely]] { - append_sample_marker(0b00000100, has_ts); // NaN - append_timestamp_if_needed(sample); - return; + *out++ = static_cast((has_ts ? 0b10000000 : 0) | 0b00000100); // NaN + } else if (val == 0.0) [[unlikely]] { + *out++ = static_cast((has_ts ? 0b10000000 : 0) | 0b00000000); // Zero + } else if (std::trunc(val) == val && val > 0.0) [[likely]] { + const uint64_t uval = static_cast(val); + if (uval <= std::numeric_limits::max()) { + const auto v = static_cast(uval); + out = write_marker_and_value(out, 0b00000001, has_ts, v); + } else if (uval <= std::numeric_limits::max()) { + const auto v = static_cast(uval); + out = write_marker_and_value(out, 0b00000010, has_ts, v); + } else if (uval <= std::numeric_limits::max()) { + const auto v = static_cast(uval); + out = write_marker_and_value(out, 0b00000011, has_ts, v); + } else { + out = write_marker_and_value(out, 0b00001001, has_ts, val); // double + } + } else { + float f = static_cast(val); + if (static_cast(f) == val) [[unlikely]] { + out = write_marker_and_value(out, 0b00001000, has_ts, f); // float + } else { + out = write_marker_and_value(out, 0b00001001, has_ts, val); // double + } } - if (val == 0.0) [[unlikely]] { - append_sample_marker(0b00000000, has_ts); // zero - append_timestamp_if_needed(sample); - return; + if (has_ts) { + const auto ts = sample.sample.timestamp(); + std::memcpy(out, &ts, sizeof(ts)); + out += sizeof(ts); } - if (std::trunc(val) == val && val > 0.0) [[likely]] { - append_integer_sample(sample, val); - } else { - append_floating_sample(sample, val); - } + const uint32_t written = static_cast(out - start); + bytes_buffer_.resize(offset + written); } private: - BareBones::Vector metric_buffer_; BareBones::Vector bytes_buffer_; - PROMPP_ALWAYS_INLINE void append_sample_marker(uint8_t type, bool has_ts) noexcept { bytes_buffer_.push_back((has_ts ? 0b10000000 : 0) | type); } - - PROMPP_ALWAYS_INLINE void append_timestamp_if_needed(const MarkedSample& sample) noexcept { - if (sample.has_ts) { - append_value(sample.sample.timestamp()); - } - } - - PROMPP_ALWAYS_INLINE void append_integer_sample(const MarkedSample& sample, double val) noexcept { - const bool has_ts = sample.has_ts; - const uint64_t uval = static_cast(val); - if (uval <= std::numeric_limits::max()) { - append_sample_marker(0b00000001, has_ts); - append_timestamp_if_needed(sample); - append_value(static_cast(uval)); - } else if (uval <= std::numeric_limits::max()) { - append_sample_marker(0b00000010, has_ts); - append_timestamp_if_needed(sample); - append_value(static_cast(uval)); - } else if (uval <= std::numeric_limits::max()) { - append_sample_marker(0b00000011, has_ts); - append_timestamp_if_needed(sample); - append_value(static_cast(uval)); - } else { - append_floating_sample(sample, val); - } - } - - PROMPP_ALWAYS_INLINE void append_floating_sample(const MarkedSample& sample, double val) noexcept { - const bool has_ts = sample.has_ts; - float f = static_cast(val); - if (static_cast(f) == val) [[unlikely]] { - append_sample_marker(0b00001000, has_ts); // float32 - append_timestamp_if_needed(sample); - append_value(f); - } else { - append_sample_marker(0b00001001, has_ts); // double - append_timestamp_if_needed(sample); - append_value(val); - } - } - template - PROMPP_ALWAYS_INLINE void append_value(T val) noexcept { - auto p = reinterpret_cast(&val); - bytes_buffer_.push_back(p, p + sizeof(T)); + PROMPP_ALWAYS_INLINE static char* write_marker_and_value(char* out, uint8_t marker, bool has_ts, const T& val) noexcept { + *out++ = static_cast((has_ts ? 0b10000000 : 0) | marker); + std::memcpy(out, &val, sizeof(T)); + return out + sizeof(T); } PROMPP_ALWAYS_INLINE static uint8_t encode_size(uint32_t v) noexcept { @@ -652,246 +581,234 @@ class Scraper { class MetadataMarkupBuffer : public MarkupBuffer { public: + using Base = MarkupBuffer; + using Iterator = typename Base::Iterator; + using IteratorSentinel = typename Base::IteratorSentinel; + + [[nodiscard]] Iterator begin(std::string_view buffer) const noexcept { return {typename Base::Context{buffer}, this->buffer_.data(), this->items_count()}; } + [[nodiscard]] static IteratorSentinel end() noexcept { return {}; } + PROMPP_ALWAYS_INLINE void add(MarkedString metric_name, MarkedString text, Prometheus::MetadataType type) noexcept { - ++this->items_count_; - - const auto offset = this->buffer_.size(); - this->buffer_.resize(offset + sizeof(MarkedMetadata)); - new (reinterpret_cast(this->buffer_.data() + offset)) MarkedMetadata{ - .metric_name = metric_name, - .text = text, - .type = type, - }; + this->buffer_.emplace_back(metric_name, text, type); } - }; - class MetricParser { - public: - MetricParser(Parser& parser, - MetricMarkupBuffer& markup_buffer, - BareBones::Vector& labels, - uint32_t global_offset, - Primitives::Timestamp timestamp) - : parser_(parser), markup_buffer_(markup_buffer), labels_(labels), sample_{.sample = {timestamp, 0.0}} { - markup_buffer_.add_metric(global_offset); + void remove_item() noexcept { this->buffer_.pop_back(); } + void initialize(size_t reserve) noexcept { + this->buffer_.clear(); + this->buffer_.reserve(reserve); } + }; - [[nodiscard]] Error parse() noexcept { - // ZoneScopedN("MetricParser::parse"); - labels_.clear(); - - bool have_metric_name = false; - auto& tokenizer = parser_.tokenizer(); - - if (tokenizer.token() == Token::kMetricName) [[likely]] { - labels_.push_back(MarkedLabel{.value = MarkedString::create(tokenizer.token_str(), tokenizer.buffer())}); - - have_metric_name = true; - tokenizer.next_non_whitespace(); - } else if (tokenizer.token() == Token::kWhitespace) [[likely]] { - tokenizer.next(); - } - - if (tokenizer.token() == Token::kBraceOpen) [[likely]] { - if (const auto error = tokenize_label_set(have_metric_name); error != Error::kNoError) { - return error; - } - - tokenizer.next_non_whitespace(); - } else if (!parser_.is_value_token()) [[unlikely]] { - return Error::kUnexpectedToken; - } - - if (!have_metric_name) [[unlikely]] { - return Error::kNoMetricName; - } - - // sort - { - const auto it = std::remove_if(labels_.begin(), labels_.end(), [](const MarkedLabel& label) { return label.value.is_empty(); }); - labels_.erase(it, labels_.end()); + [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_metadata() { + static constexpr auto get_metadata_type = [](Token token) PROMPP_LAMBDA_INLINE { + if (token == Token::kHelp) { + return Prometheus::MetadataType::kHelp; } - - std::sort(labels_.begin(), labels_.end(), [buffer = tokenizer.buffer()](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { - return a.name.view(buffer) < b.name.view(buffer); - }); - - // hash - { - BareBones::XXHash3 hash; - for (const auto& label : labels_) { - hash.extend(label.name.view(tokenizer.buffer()), label.value.view(tokenizer.buffer())); - } - markup_buffer_.add_hash(hash.hash()); + if (token == Token::kType) { + return Prometheus::MetadataType::kType; } - // encode count - { - markup_buffer_.add_count(labels_.size()); - } + return Prometheus::MetadataType::kUnit; + }; - // encode labels - for (const auto& label : labels_) { - markup_buffer_.add_label(label); - } + auto& tokenizer = parser_.tokenizer(); + const auto type = tokenizer.token(); - return parse_metric_suffix(); + if (tokenizer.next_non_whitespace() != Token::kMetricName) [[unlikely]] { + return Error::kUnexpectedToken; } - private: - Parser& parser_; - MetricMarkupBuffer& markup_buffer_; - BareBones::Vector& labels_; - MarkedSample sample_; - - [[nodiscard]] Error tokenize_label_set(bool& have_metric_name) noexcept { - auto& tokenizer = parser_.tokenizer(); - tokenizer.next_non_whitespace(); + auto metric_name = tokenizer.token_str(); + Prometheus::textparse::unquote(metric_name); - while (tokenizer.token() != Token::kBraceClose) { - MarkedLabel label; - if (const auto error = get_label_name(label.name); error != Error::kNoError) [[unlikely]] { - return error; - } + if (tokenizer.next_non_whitespace() != Token::kText) [[unlikely]] { + return Error::kUnexpectedToken; + } - if (tokenizer.next_non_whitespace() == Token::kEqual) [[likely]] { - if (tokenizer.next_non_whitespace() != Token::kLabelValue) [[unlikely]] { - return Error::kUnexpectedToken; - } + const auto text = tokenizer.token_str(); + if (const auto token = tokenizer.next_non_whitespace(); !BareBones::is_in(token, Token::kLinebreak, Token::kEOF)) [[unlikely]] { + return Error::kUnexpectedToken; + } - if (const auto error = get_quoted_value(label.value); error != Error::kNoError) [[unlikely]] { - return error; - } + if (type == Token::kHelp && !simdutf::validate_utf8(text.data(), text.size())) [[unlikely]] { + return Error::kInvalidUtf8; + } - labels_.push_back(label); + const auto buffer = tokenizer.buffer(); + metadata_buffer_.add(MarkedString::create(metric_name, buffer), MarkedString::create(text, buffer), get_metadata_type(type)); + return Error::kNoError; + } - tokenizer.next(); - } else { - if (!have_metric_name) [[unlikely]] { - labels_.push_back(MarkedLabel{.value = label.name}); + [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_metric() { + labels_.clear(); - have_metric_name = true; - } else { - return Error::kUnexpectedToken; - } - } + bool have_metric_name = false; + auto& tokenizer = parser_.tokenizer(); - if (tokenizer.token() != Token::kComma && tokenizer.token() != Token::kWhitespace) { - break; - } + metric_buffer_.add_metric(tokenizer.token_str().data() - tokenizer.buffer().data()); - tokenizer.next_non_whitespace(); - } + if (tokenizer.token() == Token::kMetricName) [[likely]] { + labels_.push_back(MarkedLabel{.value = MarkedString::create(tokenizer.token_str(), tokenizer.buffer())}); - return tokenizer.token() == Token::kBraceClose ? Error::kNoError : Error::kUnexpectedToken; + have_metric_name = true; + tokenizer.next_non_whitespace(); + } else if (tokenizer.token() == Token::kWhitespace) [[likely]] { + tokenizer.next(); } - [[nodiscard]] Error get_label_name(MarkedString& label_name) const noexcept { - auto& tokenizer = parser_.tokenizer(); - - if (tokenizer.token() == Token::kLabelName) [[likely]] { - label_name = MarkedString::create(tokenizer.token_str(), tokenizer.buffer()); - return Error::kNoError; - } - if (tokenizer.token() == Token::kQuotedString) { - return get_quoted_value(label_name); + if (tokenizer.token() == Token::kBraceOpen) [[likely]] { + if (const auto error = tokenize_label_set(have_metric_name); error != Error::kNoError) { + return error; } + tokenizer.next_non_whitespace(); + } else if (!parser_.is_value_token()) [[unlikely]] { return Error::kUnexpectedToken; } - [[nodiscard]] Error get_quoted_value(MarkedString& string) const noexcept { - auto& tokenizer = parser_.tokenizer(); - - auto value = tokenizer.token_str(); - Prometheus::textparse::unquote(value); + if (!have_metric_name) [[unlikely]] { + return Error::kNoMetricName; + } - auto copy_to = const_cast(value.data()); - Prometheus::textparse::unescape_label_value(value, [©_to](const std::string_view& piece_of_string) { - if (copy_to != piece_of_string.data()) [[unlikely]] { - memmove(copy_to, piece_of_string.data(), piece_of_string.size()); - } + // sort + { + const auto it = std::remove_if(labels_.begin(), labels_.end(), [](const MarkedLabel& label) { return label.value.is_empty(); }); + labels_.erase(it, labels_.end()); + } - copy_to += piece_of_string.size(); - }); - value.remove_suffix(value.size() - (copy_to - value.data())); + std::sort(labels_.begin(), labels_.end(), [buffer = tokenizer.buffer()](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { + return a.name.view(buffer) < b.name.view(buffer); + }); - if (!simdutf::validate_utf8(value.data(), value.size())) [[unlikely]] { - return Error::kInvalidUtf8; + // hash + { + BareBones::XXHash3 hash; + for (const auto& label : labels_) { + hash.extend(label.name.view(tokenizer.buffer()), label.value.view(tokenizer.buffer())); } + metric_buffer_.add_hash(hash.hash()); + } - string = MarkedString::create(value, tokenizer.buffer()); - return Error::kNoError; + // encode count + { + metric_buffer_.add_count(labels_.size()); } - [[nodiscard]] Error parse_metric_suffix() noexcept { - if (!parser_.is_value_token()) [[unlikely]] { - return Error::kUnexpectedToken; - } + // encode labels + for (const auto& label : labels_) { + metric_buffer_.add_label(label); + } + + return parse_metric_suffix(); + } - if (const auto error = parse_sample(); error != Error::kNoError) { + [[nodiscard]] PROMPP_ALWAYS_INLINE Error tokenize_label_set(bool& have_metric_name) noexcept { + auto& tokenizer = parser_.tokenizer(); + tokenizer.next_non_whitespace(); + + while (tokenizer.token() != Token::kBraceClose) { + MarkedLabel label; + if (const auto error = get_label_name(label.name); error != Error::kNoError) [[unlikely]] { return error; } - markup_buffer_.add_sample(sample_); + if (tokenizer.next_non_whitespace() == Token::kEqual) [[likely]] { + if (tokenizer.next_non_whitespace() != Token::kLabelValue) [[unlikely]] { + return Error::kUnexpectedToken; + } + + if (const auto error = get_quoted_value(label.value); error != Error::kNoError) [[unlikely]] { + return error; + } - return parser_.validate_parse_sample_result(); - } + labels_.push_back(label); - [[nodiscard]] Error parse_sample() noexcept { - auto& tokenizer = parser_.tokenizer(); + tokenizer.next(); + } else { + if (!have_metric_name) [[unlikely]] { + labels_.push_back(MarkedLabel{.value = label.name}); - if (!parse_numeric_value(tokenizer.token_str(), sample_.sample.value())) [[unlikely]] { - return Error::kInvalidValue; + have_metric_name = true; + } else { + return Error::kUnexpectedToken; + } } - if (std::isnan(sample_.sample.value())) [[unlikely]] { - sample_.sample.value() = Prometheus::kNormalNan; + + if (tokenizer.token() != Token::kComma && tokenizer.token() != Token::kWhitespace) { + break; } tokenizer.next_non_whitespace(); - - return parser_.parse_timestamp(sample_.sample.timestamp(), sample_.has_ts); } - }; - [[nodiscard]] Error parse_metadata() { - static constexpr auto get_metadata_type = [](Token token) PROMPP_LAMBDA_INLINE { - if (token == Token::kHelp) { - return Prometheus::MetadataType::kHelp; - } - if (token == Token::kType) { - return Prometheus::MetadataType::kType; - } + return tokenizer.token() == Token::kBraceClose ? Error::kNoError : Error::kUnexpectedToken; + } - return Prometheus::MetadataType::kUnit; - }; + [[nodiscard]] PROMPP_ALWAYS_INLINE Error get_label_name(MarkedString& label_name) const noexcept { + auto& tokenizer = parser_.tokenizer(); + + if (tokenizer.token() == Token::kLabelName) [[likely]] { + label_name = MarkedString::create(tokenizer.token_str(), tokenizer.buffer()); + return Error::kNoError; + } + if (tokenizer.token() == Token::kQuotedString) { + return get_quoted_value(label_name); + } + + return Error::kUnexpectedToken; + } + [[nodiscard]] PROMPP_ALWAYS_INLINE Error get_quoted_value(MarkedString& string) const noexcept { auto& tokenizer = parser_.tokenizer(); - const auto type = tokenizer.token(); - if (tokenizer.next_non_whitespace() != Token::kMetricName) [[unlikely]] { - return Error::kUnexpectedToken; + auto value = tokenizer.token_str(); + Prometheus::textparse::unquote(value); + + auto copy_to = const_cast(value.data()); + Prometheus::textparse::unescape_label_value(value, [©_to](const std::string_view& piece_of_string) { + if (copy_to != piece_of_string.data()) [[unlikely]] { + memmove(copy_to, piece_of_string.data(), piece_of_string.size()); + } + + copy_to += piece_of_string.size(); + }); + value.remove_suffix(value.size() - (copy_to - value.data())); + + if (!simdutf::validate_utf8(value.data(), value.size())) [[unlikely]] { + return Error::kInvalidUtf8; } - auto metric_name = tokenizer.token_str(); - Prometheus::textparse::unquote(metric_name); + string = MarkedString::create(value, tokenizer.buffer()); + return Error::kNoError; + } - if (tokenizer.next_non_whitespace() != Token::kText) [[unlikely]] { + [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_metric_suffix() noexcept { + if (!parser_.is_value_token()) [[unlikely]] { return Error::kUnexpectedToken; } - const auto text = tokenizer.token_str(); - if (const auto token = tokenizer.next_non_whitespace(); !BareBones::is_in(token, Token::kLinebreak, Token::kEOF)) [[unlikely]] { - return Error::kUnexpectedToken; + MarkedSample sample{.sample = {default_timestamp_, {}}}; + if (const auto error = parse_sample(sample); error != Error::kNoError) { + return error; } + metric_buffer_.add_sample(sample); - if (type == Token::kHelp && !simdutf::validate_utf8(text.data(), text.size())) [[unlikely]] { - return Error::kInvalidUtf8; + return parser_.validate_parse_sample_result(); + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_sample(MarkedSample& sample) noexcept { + auto& tokenizer = parser_.tokenizer(); + + if (!parse_numeric_value(tokenizer.token_str(), sample.sample.value())) [[unlikely]] { + return Error::kInvalidValue; + } + if (std::isnan(sample.sample.value())) [[unlikely]] { + sample.sample.value() = Prometheus::kNormalNan; } - const auto buffer = tokenizer.buffer(); - metadata_buffer_.add(MarkedString::create(metric_name, buffer), MarkedString::create(text, buffer), get_metadata_type(type)); - return Error::kNoError; + tokenizer.next_non_whitespace(); + + return parser_.parse_timestamp(sample.sample.timestamp(), sample.has_ts); } Parser parser_; From df0808dcef0a9c1c4878156c09181c055df7bb67 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 13 May 2026 10:21:57 +0000 Subject: [PATCH 47/48] scraper final --- pp/wal/hashdex/scraper/encoding.h | 370 +++++++++++++ pp/wal/hashdex/scraper/encoding_tests.cpp | 209 ++++++++ pp/wal/hashdex/scraper/marked.h | 218 ++++++++ pp/wal/hashdex/scraper/marked_common.h | 59 ++ pp/wal/hashdex/scraper/scraper.h | 626 +++------------------- 5 files changed, 939 insertions(+), 543 deletions(-) create mode 100644 pp/wal/hashdex/scraper/encoding.h create mode 100644 pp/wal/hashdex/scraper/encoding_tests.cpp create mode 100644 pp/wal/hashdex/scraper/marked.h create mode 100644 pp/wal/hashdex/scraper/marked_common.h diff --git a/pp/wal/hashdex/scraper/encoding.h b/pp/wal/hashdex/scraper/encoding.h new file mode 100644 index 0000000000..9dedf4c7ba --- /dev/null +++ b/pp/wal/hashdex/scraper/encoding.h @@ -0,0 +1,370 @@ +#pragma once + +#include +#include + +#include "bare_bones/bit.h" +#include "marked_common.h" +#include "primitives/sample.h" +#include "prometheus/value.h" + +namespace PromPP::WAL::hashdex::scraper::encoding { +enum class SampleValueType : uint8_t { kUint32 = 0b0000'0000, kDouble, kUint8, kUint16, kFloat, kZero, kNaN }; + +struct LayoutMarker { + bool has_ts : 1; + uint8_t length_bytes : 3; + SampleValueType sample_value_type : 4; + + [[nodiscard]] PROMPP_LAMBDA_INLINE bool has_timestamp() const noexcept { return has_ts; } + + [[nodiscard]] PROMPP_LAMBDA_INLINE uint8_t size_length_in_bytes() const noexcept { return length_bytes; } + + [[nodiscard]] PROMPP_LAMBDA_INLINE SampleValueType value_type() const noexcept { return sample_value_type; } + + static PROMPP_LAMBDA_INLINE LayoutMarker make(bool has_ts, uint32_t labels_count, SampleValueType value_type) noexcept { + const uint8_t bytes_for_count = BareBones::Bit::to_ceil_bytes(std::bit_width(labels_count)); + return LayoutMarker{.has_ts = has_ts, .length_bytes = bytes_for_count, .sample_value_type = value_type}; + } +}; + +class SampleCodec { + public: + static constexpr size_t kMaximumEncodingSize = sizeof(Primitives::Sample); + + static char* encode(char* out, const LayoutMarker layout, Primitives::Sample sample) { + using encoding::SampleValueType; + + const double val = sample.value(); + if (const auto type = layout.value_type(); type == SampleValueType::kUint32) [[likely]] { + out = write_value(out, static_cast(val)); + } else if (type == SampleValueType::kDouble) [[likely]] { + out = write_value(out, val); + } else if (type == SampleValueType::kUint8) [[unlikely]] { + out = write_value(out, static_cast(val)); + } else if (type == SampleValueType::kUint16) [[unlikely]] { + out = write_value(out, static_cast(val)); + } else if (type == SampleValueType::kFloat) [[unlikely]] { + out = write_value(out, static_cast(val)); + } + + if (layout.has_timestamp()) [[unlikely]] { + out = write_value(out, sample.timestamp()); + } + + return out; + } + + struct DecodeResult { + const char* next; + Primitives::Sample sample; + }; + + static DecodeResult decode(const char* in, const LayoutMarker layout, int64_t default_ts) { + using encoding::SampleValueType; + + DecodeResult result{}; + + double& val = result.sample.value(); + uint64_t chunk; + std::memcpy(&chunk, in, sizeof(chunk)); + + if (const auto type = layout.value_type(); type == SampleValueType::kUint32) [[likely]] { + val = static_cast(static_cast(chunk)); + in += sizeof(uint32_t); + } else if (type == SampleValueType::kDouble) [[likely]] { + val = std::bit_cast(chunk); + in += sizeof(double); + } else if (type == SampleValueType::kUint8) [[unlikely]] { + val = static_cast(static_cast(chunk)); + in += sizeof(uint8_t); + } else if (type == SampleValueType::kUint16) [[unlikely]] { + val = static_cast(static_cast(chunk)); + in += sizeof(uint16_t); + } else if (type == SampleValueType::kFloat) [[unlikely]] { + val = static_cast(std::bit_cast(static_cast(chunk))); + in += sizeof(float); + } else if (type == SampleValueType::kZero) [[unlikely]] { + val = 0.0; + } else { + val = Prometheus::kNormalNan; + } + + if (auto& timestamp = result.sample.timestamp(); layout.has_timestamp()) [[unlikely]] { + std::memcpy(×tamp, in, sizeof(timestamp)); + in += sizeof(timestamp); + } else { + timestamp = default_ts; + } + + result.next = in; + + return result; + } + + [[nodiscard]] static SampleValueType value_type(const double val) noexcept { + if (std::isnan(val)) [[unlikely]] { + return SampleValueType::kNaN; + } + + if (val == 0.0) [[unlikely]] { + return SampleValueType::kZero; + } + + if (std::trunc(val) == val && val > 0.0 && val <= std::numeric_limits::max()) [[likely]] { + const auto uval = static_cast(val); + if (uval <= std::numeric_limits::max()) { + return SampleValueType::kUint8; + } + if (uval <= std::numeric_limits::max()) { + return SampleValueType::kUint16; + } + return SampleValueType::kUint32; + } + + if (const auto f = static_cast(val); static_cast(f) == val) [[unlikely]] { + return SampleValueType::kFloat; + } + + return SampleValueType::kDouble; + } + + private: + template + PROMPP_ALWAYS_INLINE static char* write_value(char* out, const T& val) noexcept { + std::memcpy(out, &val, sizeof(T)); + return out + sizeof(T); + } +}; + +class LabelCodec { + public: + static constexpr size_t kMaximumEncodingSize = sizeof(uint8_t) + 4 * sizeof(uint32_t); + + static char* encode(char* out, const MarkedLabel label) noexcept { + if (label.name.offset == 0 && label.name.length == 0) [[likely]] { + return encode_value_only(out, label.value.offset, label.value.length); + } + + if ((label.name.offset | label.name.length | label.value.offset | label.value.length) <= 0xFF) [[likely]] { + return encode_4_bytes(out, label.name.offset, label.name.length, label.value.offset, label.value.length); + } + + return encode_generic(out, label.name.offset, label.name.length, label.value.offset, label.value.length); + } + + struct DecodeResult { + const char* next; + MarkedLabel label; + }; + + static DecodeResult decode(const char* in) noexcept { + uint64_t chunk; + std::memcpy(&chunk, in, sizeof(chunk)); + const auto layout = static_cast(chunk); + + if (layout == 0b01010101) [[likely]] { + return decode_4_bytes(in, chunk); + } + + if ((layout & 0x0F) == 0) [[likely]] { + return decode_value_only(in, chunk, layout); + } + + return decode_generic(++in, layout); + } + + private: + static PROMPP_ALWAYS_INLINE char* encode_value_only(char* out, const uint32_t label_value_offset, const uint32_t label_value_length) noexcept { + char* start = out++; + + const uint8_t sz2 = push_and_encode(out, label_value_offset); + out += szm_[sz2]; + const uint8_t sz3 = push_and_encode(out, label_value_length); + out += szm_[sz3]; + + *start = (sz2 << 4) | (sz3 << 6); + + return out; + } + + static PROMPP_ALWAYS_INLINE char* encode_4_bytes(char* out, + const uint8_t label_name_offset, + const uint8_t label_name_length, + const uint8_t label_value_offset, + const uint8_t label_value_length) noexcept { + const uint64_t chunk = static_cast(0b01010101) | static_cast(label_name_offset) << 8 | static_cast(label_name_length) << 16 | + static_cast(label_value_offset) << 24 | static_cast(label_value_length) << 32; + + std::memcpy(out, &chunk, sizeof(chunk)); + return out + 5; + } + + static PROMPP_ALWAYS_INLINE char* encode_generic(char* out, + const uint32_t label_name_offset, + const uint32_t label_name_length, + const uint32_t label_value_offset, + const uint32_t label_value_length) noexcept { + char* start = out++; + const uint8_t sz0 = push_and_encode(out, label_name_offset); + out += szm_[sz0]; + const uint8_t sz1 = push_and_encode(out, label_name_length); + out += szm_[sz1]; + const uint8_t sz2 = push_and_encode(out, label_value_offset); + out += szm_[sz2]; + const uint8_t sz3 = push_and_encode(out, label_value_length); + out += szm_[sz3]; + + *start = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); + + return out; + } + + static PROMPP_ALWAYS_INLINE DecodeResult decode_4_bytes(const char* in, const uint64_t chunk) noexcept { + const auto name_off = static_cast(chunk >> 8); + const auto name_len = static_cast(chunk >> 16); + const auto value_off = static_cast(chunk >> 24); + const auto value_len = static_cast(chunk >> 32); + + return DecodeResult{.next = in + 5, .label = {.name = {.offset = name_off, .length = name_len}, .value = {.offset = value_off, .length = value_len}}}; + } + + static PROMPP_ALWAYS_INLINE DecodeResult decode_value_only(const char* in, uint64_t chunk, const uint8_t layout) noexcept { + const uint8_t sz2 = (layout >> 4) & 0b11; + const uint8_t sz3 = (layout >> 6) & 0b11; + + chunk >>= 8; + size_t used = 1; + + uint32_t value_off = 0; + uint32_t value_len = 0; + + if (sz2 == 0b01) [[likely]] { + value_off = static_cast(chunk); + used += sizeof(uint8_t); + chunk >>= BareBones::Bit::to_bits(sizeof(uint8_t)); + } else if (sz2 == 0b10) [[unlikely]] { + value_off = static_cast(chunk); + used += sizeof(uint16_t); + chunk >>= BareBones::Bit::to_bits(sizeof(uint16_t)); + } else if (sz2 == 0b11) [[unlikely]] { + value_off = static_cast(chunk); + used += sizeof(uint32_t); + chunk >>= BareBones::Bit::to_bits(sizeof(uint32_t)); + } + if (sz3 == 0b01) [[likely]] { + value_len = static_cast(chunk); + used += sizeof(uint8_t); + } else if (sz3 == 0b10) [[unlikely]] { + value_len = static_cast(chunk); + used += sizeof(uint16_t); + } else if (sz3 == 0b11) [[unlikely]] { + if (used + sizeof(uint32_t) <= sizeof(chunk)) [[likely]] { + value_len = static_cast(chunk); + } else [[unlikely]] { + std::memcpy(&value_len, in + used, sizeof(value_len)); + } + used += sizeof(value_len); + } + + return DecodeResult{.next = in + used, .label = {.name = {.offset = 0, .length = 0}, .value = {.offset = value_off, .length = value_len}}}; + } + + static PROMPP_ALWAYS_INLINE DecodeResult decode_generic(const char* in, const uint8_t layout) noexcept { + const uint8_t sz0 = layout & 0b11; + const uint8_t sz1 = (layout >> 2) & 0b11; + const uint8_t sz2 = (layout >> 4) & 0b11; + const uint8_t sz3 = (layout >> 6) & 0b11; + + const uint32_t name_off = read_val_partial(in, sz0); + const uint32_t name_len = read_val_partial(in, sz1); + const uint32_t value_off = read_val_partial(in, sz2); + const uint32_t value_len = read_val_partial(in, sz3); + + return DecodeResult{.next = in, .label = {.name = {.offset = name_off, .length = name_len}, .value = {.offset = value_off, .length = value_len}}}; + } + + static PROMPP_ALWAYS_INLINE uint8_t push_and_encode(char* out, uint32_t v) noexcept { + if (v == 0) [[unlikely]] { + return 0b00; + } + std::memcpy(out, &v, sizeof(v)); + if (v <= 0xFF) [[likely]] { + return 0b01; + } + if (v <= 0xFFFF) [[unlikely]] { + return 0b10; + } + + return 0b11; + } + + template + static PROMPP_ALWAYS_INLINE uint32_t read_val_typed(const char*& p) noexcept { + T v; + std::memcpy(&v, p, sizeof(T)); + p += sizeof(T); + return static_cast(v); + } + + static PROMPP_ALWAYS_INLINE uint32_t read_val_partial(const char*& p, uint8_t sz) noexcept { + switch (sz) { + case 0b00: { + return 0; + } + case 0b01: { + return read_val_typed(p); + } + case 0b10: { + return read_val_typed(p); + } + default: { + return read_val_typed(p); + } + } + } + + static constexpr uint8_t szm_[4] = {0, 1, 2, 4}; +}; + +class LayoutCountCodec { + public: + static constexpr size_t kMaximumEncodingSize = sizeof(uint64_t); + + static PROMPP_ALWAYS_INLINE char* encode(char* out, const LayoutMarker layout, const uint32_t count) noexcept { + uint64_t chunk = 0; + std::memcpy(&chunk, &layout, sizeof(layout)); + chunk |= static_cast(count) << 8; + std::memcpy(out, &chunk, sizeof(chunk)); + + const uint32_t bytes_written = sizeof(layout) + layout.size_length_in_bytes(); + return out + bytes_written; + } + + struct DecodeResult { + const char* next; + LayoutMarker layout; + uint32_t count; + }; + + static PROMPP_ALWAYS_INLINE DecodeResult decode(const char* in) noexcept { + uint64_t chunk; + std::memcpy(&chunk, in, sizeof(chunk)); + + LayoutMarker layout{}; + std::memcpy(&layout, &chunk, sizeof(layout)); + + chunk >>= 8; + const uint64_t mask = (1ULL << BareBones::Bit::to_bits(layout.size_length_in_bytes())) - 1; + + auto labels_count = static_cast(chunk & mask); + + return {in + sizeof(layout) + layout.size_length_in_bytes(), layout, labels_count}; + } +}; + +static constexpr uint32_t metric_maximum_encoding_size(uint32_t labels_count) noexcept { + return LayoutCountCodec::kMaximumEncodingSize + LabelCodec::kMaximumEncodingSize * labels_count + SampleCodec::kMaximumEncodingSize; +} + +} // namespace PromPP::WAL::hashdex::scraper::encoding \ No newline at end of file diff --git a/pp/wal/hashdex/scraper/encoding_tests.cpp b/pp/wal/hashdex/scraper/encoding_tests.cpp new file mode 100644 index 0000000000..6a5d021e5b --- /dev/null +++ b/pp/wal/hashdex/scraper/encoding_tests.cpp @@ -0,0 +1,209 @@ +#include + +#include +#include +#include +#include + +#include "encoding.h" +#include "primitives/sample.h" +#include "prometheus/value.h" + +namespace { + +using PromPP::Primitives::Sample; +using PromPP::Primitives::Timestamp; +using PromPP::WAL::hashdex::scraper::encoding::LabelCodec; +using PromPP::WAL::hashdex::scraper::encoding::LayoutMarker; +using PromPP::WAL::hashdex::scraper::encoding::SampleCodec; +using PromPP::WAL::hashdex::scraper::encoding::SampleValueType; + +struct ValueTypeCase { + double input; + SampleValueType expected; +}; + +class ValueTypeFixture : public testing::TestWithParam {}; + +TEST_P(ValueTypeFixture, ClassifyValueCorrectly) { + // Arrange + + // Act + const auto actual = SampleCodec::value_type(GetParam().input); + + // Assert + EXPECT_EQ(actual, GetParam().expected); +} + +INSTANTIATE_TEST_SUITE_P(ValueTypeTests, + ValueTypeFixture, + testing::Values(ValueTypeCase{.input = PromPP::Prometheus::kNormalNan, .expected = SampleValueType::kNaN}, + ValueTypeCase{.input = PromPP::Prometheus::kStaleNan, .expected = SampleValueType::kNaN}, + ValueTypeCase{.input = std::numeric_limits::quiet_NaN(), .expected = SampleValueType::kNaN}, + + ValueTypeCase{.input = 0.0, .expected = SampleValueType::kZero}, + ValueTypeCase{.input = -0.0, .expected = SampleValueType::kZero}, + + ValueTypeCase{.input = 1.0, .expected = SampleValueType::kUint8}, + ValueTypeCase{.input = 255.0, .expected = SampleValueType::kUint8}, + + ValueTypeCase{.input = 256.0, .expected = SampleValueType::kUint16}, + ValueTypeCase{.input = 65535.0, .expected = SampleValueType::kUint16}, + + ValueTypeCase{.input = 65536.0, .expected = SampleValueType::kUint32}, + ValueTypeCase{.input = 4294967295.0, .expected = SampleValueType::kUint32}, + + ValueTypeCase{.input = 3.5, .expected = SampleValueType::kFloat}, + ValueTypeCase{.input = -42.0, .expected = SampleValueType::kFloat}, + + ValueTypeCase{.input = 1e-10, .expected = SampleValueType::kDouble}, + ValueTypeCase{.input = 3.141592653589793, .expected = SampleValueType::kDouble}, + ValueTypeCase{.input = 0.1, .expected = SampleValueType::kDouble})); + +struct SampleCodecCase { + Sample sample; + bool has_ts; +}; + +class SampleCodecFixture : public testing::TestWithParam { + protected: + static constexpr size_t kBufSize = 64; + + SampleCodec::DecodeResult encode_and_decode(const LayoutMarker layout, Sample sample, int64_t default_ts = -1) { + buf_.fill(0); + char* start = buf_.data(); + char* end = SampleCodec::encode(start, layout, sample); + + const auto res = SampleCodec::decode(start, layout, default_ts); + + EXPECT_EQ(res.next, end); + return res; + } + + std::array buf_{}; + const Timestamp default_ts_ = -1; +}; + +TEST_P(SampleCodecFixture, CorrectSample) { + // Arrange + const auto layout = LayoutMarker::make(GetParam().has_ts, 0, SampleCodec::value_type(GetParam().sample.value())); + + Sample expected = GetParam().sample; + if (!GetParam().has_ts) { + expected.timestamp() = default_ts_; + } + + // Act + const auto res = encode_and_decode(layout, GetParam().sample, default_ts_); + + // Assert + EXPECT_EQ(res.sample, expected); +} + +INSTANTIATE_TEST_SUITE_P(SampleCodecTests, + SampleCodecFixture, + testing::Values(SampleCodecCase{.sample = Sample(42, 123.0), .has_ts = true}, + SampleCodecCase{.sample = Sample(12345, std::numbers::pi), .has_ts = false}, + SampleCodecCase{.sample = Sample(111, 1.2345), .has_ts = true}, + SampleCodecCase{.sample = Sample(7, 255.0), .has_ts = true}, + SampleCodecCase{.sample = Sample(9, 65535.0), .has_ts = true}, + SampleCodecCase{.sample = Sample(222, 0.0), .has_ts = true}, + SampleCodecCase{.sample = Sample(333, PromPP::Prometheus::kNormalNan), .has_ts = true}, + SampleCodecCase{.sample = Sample(1000, 42.0), .has_ts = false}, + SampleCodecCase{.sample = Sample(2048, 4242.0), .has_ts = true})); + +class LabelCodecFixture : public testing::Test { + protected: + static constexpr size_t kBufSize = 64; + std::array buf_{}; + + LabelCodec::DecodeResult encode_and_decode(uint32_t name_off, uint32_t name_len, uint32_t value_off, uint32_t value_len) { + buf_.fill(0); + char* start = buf_.data(); + char* end = LabelCodec::encode(start, PromPP::WAL::hashdex::scraper::MarkedLabel{.name = {.offset = name_off, .length = name_len}, + .value = {.offset = value_off, .length = value_len}}); + + const auto res = LabelCodec::decode(start); + + EXPECT_EQ(res.next, end); + return res; + } +}; + +TEST_F(LabelCodecFixture, FastPath_Layout01010101) { + // Arrange + buf_.fill(0); + char* start = buf_.data(); + start[0] = static_cast(0b01010101); + start[1] = 10; + start[2] = 11; + start[3] = 12; + start[4] = 13; + + // Act + const auto res = LabelCodec::decode(buf_.data()); + + // Assert + EXPECT_EQ(res.label.name.offset, 10u); + EXPECT_EQ(res.label.name.length, 11u); + EXPECT_EQ(res.label.value.offset, 12u); + EXPECT_EQ(res.label.value.length, 13u); + EXPECT_EQ(res.next, buf_.data() + 5); +} + +TEST_F(LabelCodecFixture, SimplifiedPath_NameFieldsZero) { + // Arrange + buf_.fill(0); + uint8_t layout = 0b10010000; + buf_[0] = static_cast(layout); + buf_[1] = 42; + uint16_t len = 1234; + std::memcpy(buf_.data() + 2, &len, 2); + + // Act + const auto res = LabelCodec::decode(buf_.data()); + + // Assert + EXPECT_EQ(res.label.name.offset, 0u); + EXPECT_EQ(res.label.name.length, 0u); + EXPECT_EQ(res.label.value.offset, 42u); + EXPECT_EQ(res.label.value.length, 1234u); + EXPECT_EQ(res.next, buf_.data() + 1 + 1 + 2); +} + +struct LabelCase { + uint32_t name_off; + uint32_t name_len; + uint32_t value_off; + uint32_t value_len; +}; + +class LabelCodecParamFixture : public LabelCodecFixture, public testing::WithParamInterface {}; + +TEST_P(LabelCodecParamFixture, EncodeDecode) { + // Arrange + + // Act + const auto res = encode_and_decode(GetParam().name_off, GetParam().name_len, GetParam().value_off, GetParam().value_len); + + // Assert + EXPECT_EQ(res.label.name.offset, GetParam().name_off); + EXPECT_EQ(res.label.name.length, GetParam().name_len); + EXPECT_EQ(res.label.value.offset, GetParam().value_off); + EXPECT_EQ(res.label.value.length, GetParam().value_len); +} + +INSTANTIATE_TEST_SUITE_P(LabelCodecTests, + LabelCodecParamFixture, + testing::Values(LabelCase{.name_off = 0, .name_len = 0, .value_off = 0, .value_len = 0}, + LabelCase{.name_off = 1, .name_len = 2, .value_off = 3, .value_len = 4}, + LabelCase{.name_off = 300, .name_len = 400, .value_off = 500, .value_len = 600}, + LabelCase{.name_off = 100000, .name_len = 200000, .value_off = 300000, .value_len = 400000}, + LabelCase{.name_off = 0, .name_len = 0, .value_off = 300000, .value_len = 400000}, + LabelCase{.name_off = 0, .name_len = 0, .value_off = 1234, .value_len = 123456}, + LabelCase{.name_off = 0, .name_len = 12, .value_off = 1234, .value_len = 123456}, + LabelCase{.name_off = 0, .name_len = 12, .value_off = 1234, .value_len = 255}, + LabelCase{.name_off = 0, .name_len = 12, .value_off = 128, .value_len = 2355}, + LabelCase{.name_off = 256, .name_len = 128, .value_off = 255, .value_len = 255})); + +} // namespace \ No newline at end of file diff --git a/pp/wal/hashdex/scraper/marked.h b/pp/wal/hashdex/scraper/marked.h new file mode 100644 index 0000000000..7409b2a5e9 --- /dev/null +++ b/pp/wal/hashdex/scraper/marked.h @@ -0,0 +1,218 @@ +#pragma once + +#include + +#include "bare_bones/vector.h" +#include "encoding.h" +#include "marked_common.h" +#include "primitives/primitives.h" +#include "prometheus/metric.h" + +namespace PromPP::WAL::hashdex::scraper::inline marked { + +class Metric { + public: + using MarkedT = MarkedMetric; + + struct Context { + std::string_view buffer; + const BareBones::Memory& bytes_buffer; + Primitives::Timestamp default_timestamp{}; + }; + + Metric(const Context& ctx, const MarkedMetric* item) + : buffer_(ctx.buffer), bytes_buffer_(ctx.bytes_buffer), item_(item), default_timestamp_(ctx.default_timestamp) {} + + [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetric* item() const noexcept { return item_; } + PROMPP_ALWAYS_INLINE void set_item(const MarkedMetric* item) noexcept { item_ = item; } + + [[nodiscard]] PROMPP_ALWAYS_INLINE uint64_t hash() const noexcept { return item_->hash; } + + template + void read(Timeseries& ts) const { + const char* ptr = bytes_buffer_.control_block().data + item_->data_offset; + + const auto [next_ptr, layout, labels_count] = encoding::LayoutCountCodec::decode(ptr); + ptr = next_ptr; + + ts.label_set().resize(labels_count); + + auto label_iter = ts.label_set().begin(); + for (uint32_t i = 0; i < labels_count; ++i) { + const auto [next_ptr, label] = encoding::LabelCodec::decode(ptr); + ptr = next_ptr; + + if (const auto buf_ptr = buffer_.data() + item_->base_offset; label.name.is_reserved_name()) [[unlikely]] { + std::construct_at(label_iter++, Prometheus::kMetricLabelName, std::string_view(buf_ptr + label.value.offset, label.value.length)); + } else { + std::construct_at(label_iter++, std::string_view(buf_ptr + label.name.offset, label.name.length), + std::string_view(buf_ptr + label.value.offset, label.value.length)); + } + } + + auto [p, sample] = encoding::SampleCodec::decode(ptr, layout, default_timestamp_); + + ts.samples().emplace_back(sample); + } + + private: + std::string_view buffer_; + const BareBones::Memory& bytes_buffer_; + const MarkedMetric* item_{}; + Primitives::Timestamp default_timestamp_; +}; + +class Metadata { + public: + using MarkedT = MarkedMetadata; + + struct Context { + std::string_view buffer; + }; + + Metadata(const Context& ctx, const MarkedMetadata* item) : buffer_(ctx.buffer), item_(item) {} + + [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetadata* item() const noexcept { return item_; } + PROMPP_ALWAYS_INLINE void set_item(const MarkedMetadata* item) noexcept { item_ = item; } + + [[nodiscard]] PROMPP_ALWAYS_INLINE Prometheus::MetadataType type() const noexcept { return item_->type; } + [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view metric_name() const noexcept { return item_->metric_name.view(buffer_); } + [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view text() const noexcept { return item_->text.view(buffer_); } + + private: + std::string_view buffer_; + const MarkedMetadata* item_{}; +}; + +template +class MarkupBuffer { + public: + using MarkedT = typename T::MarkedT; + using Context = typename T::Context; + + class IteratorSentinel {}; + + class Iterator { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + Iterator(const Context& ctx, const MarkedT* ptr, uint32_t items_count) : item_(ctx, ptr), ptr_(ptr), items_count_(items_count), ctx_(ctx) {} + + [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type& operator*() const noexcept { return item_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type* operator->() const noexcept { return &item_; } + + PROMPP_ALWAYS_INLINE Iterator& operator++() noexcept { + item_.set_item(++ptr_); + --items_count_; + return *this; + } + + PROMPP_ALWAYS_INLINE Iterator operator++(int) noexcept { + auto tmp = *this; + ++(*this); + return tmp; + } + + PROMPP_ALWAYS_INLINE bool operator==(const IteratorSentinel&) const noexcept { return items_count_ == 0; } + + private: + T item_; + const MarkedT* ptr_; + uint32_t items_count_; + Context ctx_; + }; + + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t items_count() const noexcept { return buffer_.size(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return buffer_.allocated_memory(); } + + protected: + BareBones::Vector buffer_; +}; + +class MetricMarkupBuffer : public MarkupBuffer { + public: + using Base = MarkupBuffer; + using Iterator = typename Base::Iterator; + using IteratorSentinel = typename Base::IteratorSentinel; + + [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer, Primitives::Timestamp default_ts) const noexcept { + return {typename Base::Context{buffer, bytes_buffer_, default_ts}, Base::buffer_.data(), Base::items_count()}; + } + [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } + + void bytes_enlarge(uint32_t extra_bytes) noexcept { + const uint32_t offset = bytes_count(); + + bytes_buffer_.grow_to_fit_at_least(offset + extra_bytes); + + bytes_ptr_ = bytes_buffer_.control_block().data + offset; + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return Base::buffer_.allocated_memory() + bytes_buffer_.allocated_memory(); } + + PROMPP_ALWAYS_INLINE void initialize(size_t reserve_bytes) noexcept { + this->buffer_.clear(); + + const size_t bytes_buffer_reserve = (reserve_bytes / 3) * 2; + const size_t items_buffer_reserve = (bytes_buffer_reserve / 3) / sizeof(MarkedMetric); + + this->buffer_.reserve(items_buffer_reserve); + bytes_buffer_.resize_to_fit_at_least(bytes_buffer_reserve); + bytes_ptr_ = bytes_buffer_.control_block().data; + } + + PROMPP_ALWAYS_INLINE void add_hash(uint64_t hash) noexcept { this->buffer_.back().hash = hash; } + + PROMPP_ALWAYS_INLINE void add_metric(uint32_t global_offset) noexcept { + this->buffer_.push_back(MarkedMetric{.hash = {}, .base_offset = global_offset, .data_offset = bytes_count()}); + } + + void add_layout_and_count(const encoding::LayoutMarker layout, const uint32_t count) noexcept { + bytes_ptr_ = encoding::LayoutCountCodec::encode(bytes_ptr_, layout, count); + } + + void add_label(MarkedLabel label) noexcept { bytes_ptr_ = encoding::LabelCodec::encode(bytes_ptr_, label); } + + void add_sample(encoding::LayoutMarker layout, const Primitives::Sample sample) noexcept { + bytes_ptr_ = encoding::SampleCodec::encode(bytes_ptr_, layout, sample); + } + + void add_padding() noexcept { + constexpr size_t kPaddingSizeBytes = 16; + bytes_enlarge(kPaddingSizeBytes); + } + + private: + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t bytes_count() const noexcept { return bytes_ptr_ - bytes_buffer_.control_block().data; } + + BareBones::Memory bytes_buffer_; + char* bytes_ptr_{}; +}; + +class MetadataMarkupBuffer : public MarkupBuffer { + public: + using Base = MarkupBuffer; + using Iterator = typename Base::Iterator; + using IteratorSentinel = typename Base::IteratorSentinel; + + [[nodiscard]] PROMPP_ALWAYS_INLINE Iterator begin(std::string_view buffer) const noexcept { + return {typename Base::Context{buffer}, this->buffer_.data(), this->items_count()}; + } + [[nodiscard]] PROMPP_ALWAYS_INLINE static IteratorSentinel end() noexcept { return {}; } + + PROMPP_ALWAYS_INLINE void initialize(size_t reserve_bytes) noexcept { + this->buffer_.clear(); + const size_t items_buffer_reserve = reserve_bytes / sizeof(MarkedMetric); + this->buffer_.reserve(items_buffer_reserve); + } + + PROMPP_ALWAYS_INLINE void add(MarkedString metric_name, MarkedString text, Prometheus::MetadataType type) noexcept { + this->buffer_.emplace_back(metric_name, text, type); + } +}; + +} // namespace PromPP::WAL::hashdex::scraper::inline marked \ No newline at end of file diff --git a/pp/wal/hashdex/scraper/marked_common.h b/pp/wal/hashdex/scraper/marked_common.h new file mode 100644 index 0000000000..8036173543 --- /dev/null +++ b/pp/wal/hashdex/scraper/marked_common.h @@ -0,0 +1,59 @@ +#pragma once + +#include + +#include "primitives/sample.h" +#include "prometheus/metric.h" +#include "prometheus/value.h" + +namespace PromPP::WAL::hashdex::scraper::inline marked { + +#pragma pack(push, 1) +struct MarkedString { + uint32_t offset = 0; + uint32_t length = 0; + + [[nodiscard]] PROMPP_ALWAYS_INLINE static MarkedString create(std::string_view value, std::string_view buffer) noexcept { + return { + .offset = static_cast(value.data() - buffer.data()), + .length = static_cast(value.size()), + }; + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_reserved_name() const noexcept { return offset == 0 && length == 0; } + + [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_empty() const noexcept { return length == 0; } + + [[nodiscard]] std::string_view view(const std::string_view& buffer) const noexcept { + if (is_reserved_name()) [[unlikely]] { + return Prometheus::kMetricLabelName; + } + + return buffer.substr(offset, length); + } +}; + +struct MarkedLabel { + MarkedString name{}; + MarkedString value; +}; + +struct MarkedSample { + Primitives::Sample sample{}; + bool has_ts{}; +}; + +struct MarkedMetric { + uint64_t hash; + uint32_t base_offset; + uint32_t data_offset; +}; + +struct MarkedMetadata { + MarkedString metric_name{}; + MarkedString text{}; + Prometheus::MetadataType type{}; +}; +#pragma pack(pop) + +} // namespace PromPP::WAL::hashdex::scraper::inline marked \ No newline at end of file diff --git a/pp/wal/hashdex/scraper/scraper.h b/pp/wal/hashdex/scraper/scraper.h index db71bc8bf0..b3538c59ca 100644 --- a/pp/wal/hashdex/scraper/scraper.h +++ b/pp/wal/hashdex/scraper/scraper.h @@ -7,23 +7,24 @@ #include "bare_bones/algorithm.h" #include "bare_bones/vector.h" #include "bare_bones/xxhash.h" +#include "encoding.h" +#include "marked.h" #include "parser.h" #include "prometheus/hashdex.h" #include "prometheus/metric.h" #include "prometheus/textparse/escape.h" #include "prometheus/value.h" -#include "primitives/sample.h" - -// #include "tracy/Tracy.hpp" - namespace PromPP::WAL::hashdex::scraper { template class Scraper { public: - [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { + [[nodiscard]] Error parse(std::span buffer, Primitives::Timestamp default_timestamp) { + metric_buffer_.initialize(buffer.size() / 4); + metadata_buffer_.initialize(buffer.size() / 128); labels_.reserve(255); + default_timestamp_ = default_timestamp; auto& tokenizer = parser_.tokenizer(); @@ -33,6 +34,7 @@ class Scraper { switch (tokenizer.next()) { case Token::kEOF: case Token::kEOFWord: { + metric_buffer_.add_padding(); return parser_.validate_parse_result(); } @@ -54,7 +56,6 @@ class Scraper { case Token::kMetricName: case Token::kBraceOpen: { if (const auto error = parse_metric(); error != Error::kNoError) [[unlikely]] { - metric_buffer_.remove_item(); return error; } @@ -76,11 +77,11 @@ class Scraper { public: explicit MetricsWrapper(const Scraper& scraper) : scraper_(scraper) {} - [[nodiscard]] PROMPP_LAMBDA_INLINE uint32_t size() const noexcept { return scraper_.metric_buffer_.items_count(); } - [[nodiscard]] PROMPP_LAMBDA_INLINE auto begin() const noexcept { + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t size() const noexcept { return scraper_.metric_buffer_.items_count(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE auto begin() const noexcept { return scraper_.metric_buffer_.begin(scraper_.parser_.tokenizer().buffer(), scraper_.default_timestamp()); } - [[nodiscard]] PROMPP_LAMBDA_INLINE static auto end() noexcept { return MetricMarkupBuffer::end(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE static auto end() noexcept { return MetricMarkupBuffer::end(); } private: const Scraper& scraper_; @@ -90,17 +91,17 @@ class Scraper { public: explicit MetadataWrapper(const Scraper& scraper) : scraper_(scraper) {} - [[nodiscard]] PROMPP_LAMBDA_INLINE uint32_t size() const noexcept { return scraper_.metadata_buffer_.items_count(); } - [[nodiscard]] PROMPP_LAMBDA_INLINE auto begin() const noexcept { return scraper_.metadata_buffer_.begin(scraper_.parser_.tokenizer().buffer()); } - [[nodiscard]] PROMPP_LAMBDA_INLINE static auto end() noexcept { return MetadataMarkupBuffer::end(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t size() const noexcept { return scraper_.metadata_buffer_.items_count(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE auto begin() const noexcept { return scraper_.metadata_buffer_.begin(scraper_.parser_.tokenizer().buffer()); } + [[nodiscard]] PROMPP_ALWAYS_INLINE static auto end() noexcept { return MetadataMarkupBuffer::end(); } private: const Scraper& scraper_; }; - [[nodiscard]] PROMPP_LAMBDA_INLINE uint32_t size() const noexcept { return metric_buffer_.items_count(); } - [[nodiscard]] PROMPP_LAMBDA_INLINE auto begin() const noexcept { return metric_buffer_.begin(parser_.tokenizer().buffer(), default_timestamp_); } - [[nodiscard]] PROMPP_LAMBDA_INLINE static auto end() noexcept { return MetricMarkupBuffer::end(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t size() const noexcept { return metric_buffer_.items_count(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE auto begin() const noexcept { return metric_buffer_.begin(parser_.tokenizer().buffer(), default_timestamp_); } + [[nodiscard]] PROMPP_ALWAYS_INLINE static auto end() noexcept { return MetricMarkupBuffer::end(); } [[nodiscard]] PROMPP_ALWAYS_INLINE MetricsWrapper metrics() const noexcept { return MetricsWrapper{*this}; } [[nodiscard]] PROMPP_ALWAYS_INLINE MetadataWrapper metadata() const noexcept { return MetadataWrapper{*this}; } @@ -108,498 +109,13 @@ class Scraper { [[nodiscard]] PROMPP_ALWAYS_INLINE Primitives::Timestamp default_timestamp() const noexcept { return default_timestamp_; } [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { - return metric_buffer_.allocated_memory() + metadata_buffer_.allocated_memory(); + return metric_buffer_.allocated_memory() + metadata_buffer_.allocated_memory() + labels_.allocated_memory(); } private: using Token = Prometheus::textparse::Token; -#pragma pack(push, 1) - struct MarkedString { - uint32_t offset{std::numeric_limits::max()}; - uint32_t length{std::numeric_limits::max()}; - - [[nodiscard]] PROMPP_ALWAYS_INLINE static MarkedString create(const std::string_view& value, const std::string_view& buffer) noexcept { - return { - .offset = static_cast(value.data() - buffer.data()), - .length = static_cast(value.size()), - }; - } - - [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_reserved_name() const noexcept { - return offset == std::numeric_limits::max() && length == std::numeric_limits::max(); - } - - [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_empty() const noexcept { return length == 0; } - - [[nodiscard]] std::string_view view(const std::string_view& buffer) const noexcept { - if (is_reserved_name()) [[unlikely]] { - return Prometheus::kMetricLabelName; - } - - return buffer.substr(offset, length); - } - }; - - struct MarkedLabel { - MarkedString name{}; - MarkedString value; - }; - - struct MarkedSample { - Primitives::Sample sample{}; - bool has_ts{}; - }; - - struct MarkedMetric { - uint64_t hash{}; - uint32_t base_offset{}; - uint32_t data_offset{}; - }; - - struct MarkedMetadata { - MarkedString metric_name{}; - MarkedString text{}; - Prometheus::MetadataType type{}; - }; -#pragma pack(pop) - - public: - class Metric { - public: - using MarkedT = MarkedMetric; - - struct Context { - std::string_view buffer; - const BareBones::Vector& bytes_buffer; - Primitives::Timestamp default_timestamp; - }; - - Metric(const Context& ctx, const MarkedMetric* item) - : buffer_(ctx.buffer), bytes_buffer_(ctx.bytes_buffer), item_(item), default_timestamp_(ctx.default_timestamp) {} - - [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetric* item() const noexcept { return item_; } - PROMPP_ALWAYS_INLINE void set_item(const MarkedMetric* item) noexcept { item_ = item; } - - [[nodiscard]] PROMPP_ALWAYS_INLINE uint64_t hash() const noexcept { return item_->hash; } - - template - void read(Timeseries& ts) const { - const char* ptr = reinterpret_cast(bytes_buffer_.data() + item_->data_offset); - const char* base = buffer_.data() + item_->base_offset; - - // decode label count - uint32_t labels_count = decode_varint(ptr); - ts.label_set().reserve(labels_count); - - // decode label - for (uint32_t i = 0; i < labels_count; ++i) { - const uint8_t layout = static_cast(*ptr++); - const uint8_t sz0 = (layout >> 0) & 0x3; - const uint8_t sz1 = (layout >> 2) & 0x3; - const uint8_t sz2 = (layout >> 4) & 0x3; - const uint8_t sz3 = (layout >> 6) & 0x3; - - const uint32_t name_off = read_val_partial(ptr, sz0); - const uint32_t name_len = read_val_partial(ptr, sz1); - const uint32_t value_off = read_val_partial(ptr, sz2); - const uint32_t value_len = read_val_partial(ptr, sz3); - - if (name_len == 0 && name_off == 0) [[unlikely]] { - ts.label_set().append(Prometheus::kMetricLabelName, std::string_view(base + value_off, value_len)); - } else { - ts.label_set().append(std::string_view(base + name_off, name_len), std::string_view(base + value_off, value_len)); - } - } - - // decode sample - uint8_t marker = static_cast(*ptr++); - bool has_ts = (marker & 0b10000000) != 0; - uint8_t type = marker & 0b01111111; - - Primitives::Sample sample{}; - sample.timestamp() = default_timestamp_; - - double val; - - switch (type) { - case 0b00000000: // zero - val = 0.0; - break; - - case 0b00000001: { // uint8 - val = static_cast(*ptr++); - break; - } - - case 0b00000010: { // uint16 - uint16_t x = static_cast(ptr[0]) | (static_cast(ptr[1]) << 8); - ptr += 2; - val = static_cast(x); - break; - } - - case 0b00000011: { // uint32 - uint32_t x; - std::memcpy(&x, ptr, sizeof(x)); - ptr += 4; - val = static_cast(x); - break; - } - - case 0b00000100: // NaN (normal) - val = Prometheus::kNormalNan; - break; - - case 0b00001000: { // float32 - float x; - std::memcpy(&x, ptr, sizeof(x)); - ptr += sizeof(x); - val = static_cast(x); - break; - } - - case 0b00001001: { // double - double x; - std::memcpy(&x, ptr, sizeof(x)); - ptr += sizeof(x); - val = x; - break; - } - - default: - val = Prometheus::kStaleNan; - break; - } - - if (has_ts) [[unlikely]] { - Primitives::Timestamp ts_val; - memcpy(&ts_val, ptr, sizeof(ts_val)); - sample.timestamp() = ts_val; - } - - sample.value() = val; - ts.samples().emplace_back(sample); - } - - private: - std::string_view buffer_; - const BareBones::Vector& bytes_buffer_; - const MarkedMetric* item_{}; - Primitives::Timestamp default_timestamp_; - - PROMPP_ALWAYS_INLINE static uint32_t decode_varint(const char*& ptr) noexcept { - uint32_t b0 = static_cast(*ptr++); - if ((b0 & 0x80) == 0) [[likely]] { - return b0; - } - - uint32_t b1 = static_cast(*ptr++); - uint32_t v = (b0 & 0x7F) | ((b1 & 0x7F) << 7); - if ((b1 & 0x80) == 0) { - return v; - } - - uint32_t b2 = static_cast(*ptr++); - v |= (b2 & 0x7F) << 14; - if ((b2 & 0x80) == 0) { - return v; - } - - uint32_t b3 = static_cast(*ptr++); - v |= (b3 & 0x7F) << 21; - if ((b3 & 0x80) == 0) { - return v; - } - - uint32_t b4 = static_cast(*ptr++); - v |= (b4 & 0x0F) << 28; - return v; - } - - PROMPP_ALWAYS_INLINE - static uint32_t read_val_partial(const char*& p, uint8_t sz) noexcept { - if (sz == 0) [[likely]] { - const uint32_t v = static_cast(p[0]); - p += 1; - return v; - } - - if (sz == 1) { - const uint32_t v = static_cast(static_cast(p[0])) | (static_cast(static_cast(p[1])) << 8); - p += 2; - return v; - } - - if (sz == 2) { - const uint32_t v = static_cast(static_cast(p[0])) | (static_cast(static_cast(p[1])) << 8) | - (static_cast(static_cast(p[2])) << 16); - p += 3; - return v; - } - - uint32_t v; - std::memcpy(&v, p, 4); - p += 4; - return v; - } - }; - - class Metadata { - public: - using MarkedT = MarkedMetadata; - - struct Context { - std::string_view buffer; - }; - - Metadata(const Context& ctx, const MarkedMetadata* item) : buffer_(ctx.buffer), item_(item) {} - - [[nodiscard]] PROMPP_ALWAYS_INLINE const MarkedMetadata* item() const noexcept { return item_; } - PROMPP_ALWAYS_INLINE void set_item(const MarkedMetadata* item) noexcept { item_ = item; } - - [[nodiscard]] PROMPP_ALWAYS_INLINE Prometheus::MetadataType type() const noexcept { return item_->type; } - [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view metric_name() const noexcept { return item_->metric_name.view(buffer_); } - [[nodiscard]] PROMPP_ALWAYS_INLINE std::string_view text() const noexcept { return item_->text.view(buffer_); } - - private: - std::string_view buffer_; - const MarkedMetadata* item_{}; - }; - - template - class MarkupBuffer { - public: - using MarkedT = typename T::MarkedT; - using Context = typename T::Context; - - class IteratorSentinel {}; - - class Iterator { - public: - using iterator_category = std::forward_iterator_tag; - using value_type = T; - using difference_type = ptrdiff_t; - using pointer = value_type*; - using reference = value_type&; - - Iterator(const Context& ctx, const MarkedT* ptr, uint32_t items_count) : item_(ctx, ptr), ptr_(ptr), items_count_(items_count), ctx_(ctx) {} - - [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type& operator*() const noexcept { return item_; } - [[nodiscard]] PROMPP_ALWAYS_INLINE const value_type* operator->() const noexcept { return &item_; } - - PROMPP_ALWAYS_INLINE Iterator& operator++() noexcept { - item_.set_item(++ptr_); - --items_count_; - return *this; - } - - PROMPP_ALWAYS_INLINE Iterator operator++(int) noexcept { - auto tmp = *this; - ++(*this); - return tmp; - } - - PROMPP_ALWAYS_INLINE bool operator==(const IteratorSentinel&) const noexcept { return items_count_ == 0; } - - private: - T item_; - const MarkedT* ptr_; - uint32_t items_count_; - Context ctx_; - }; - - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t items_count() const noexcept { return buffer_.size(); } - [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return buffer_.size() * sizeof(MarkedT); } - - protected: - BareBones::Vector buffer_; - }; - - class MetricMarkupBuffer : public MarkupBuffer { - public: - using Base = MarkupBuffer; - using Iterator = typename Base::Iterator; - using IteratorSentinel = typename Base::IteratorSentinel; - - [[nodiscard]] Iterator begin(std::string_view buffer, Primitives::Timestamp default_ts) const noexcept { - return {typename Base::Context{buffer, bytes_buffer_, default_ts}, this->buffer_.data(), this->items_count()}; - } - [[nodiscard]] static IteratorSentinel end() noexcept { return {}; } - - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t bytes_count() const noexcept { return bytes_buffer_.size(); } - - [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return this->buffer_.size() * sizeof(MarkedMetric) + bytes_buffer_.size(); } - - PROMPP_ALWAYS_INLINE void remove_item() noexcept { - bytes_buffer_.resize(this->buffer_.back().data_offset); - this->buffer_.resize(this->buffer_.size() - 1); - } - - PROMPP_ALWAYS_INLINE void add_hash(uint64_t hash) noexcept { this->buffer_.back().hash = hash; } - PROMPP_ALWAYS_INLINE void add_metric(uint32_t global_offset) noexcept { - this->buffer_.push_back(MarkedMetric{.base_offset = global_offset, .data_offset = bytes_count()}); - } - - PROMPP_ALWAYS_INLINE void add_count(uint32_t count) noexcept { - constexpr uint8_t kContinueBit = 0x80; - constexpr uint8_t kValueMask = 0x7F; - - const uint32_t offset = bytes_count(); - - if (count < (1u << 7)) [[likely]] { - bytes_buffer_.resize(bytes_buffer_.size() + 1); - char* out = bytes_buffer_.data() + offset; - *out = static_cast(count); - } else if (count < (1u << 14)) { - bytes_buffer_.resize(bytes_buffer_.size() + 2); - char* out = bytes_buffer_.data() + offset; - *out++ = static_cast((count & kValueMask) | kContinueBit); - *out = static_cast(count >> 7); - } else if (count < (1u << 21)) { - bytes_buffer_.resize(bytes_buffer_.size() + 3); - char* out = bytes_buffer_.data() + offset; - *out++ = static_cast((count & kValueMask) | kContinueBit); - *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); - *out = static_cast(count >> 14); - } else if (count < (1u << 28)) { - bytes_buffer_.resize(bytes_buffer_.size() + 4); - char* out = bytes_buffer_.data() + offset; - *out++ = static_cast((count & kValueMask) | kContinueBit); - *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); - *out++ = static_cast(((count >> 14) & kValueMask) | kContinueBit); - *out = static_cast(count >> 21); - } else { - bytes_buffer_.resize(bytes_buffer_.size() + 5); - char* out = bytes_buffer_.data() + offset; - *out++ = static_cast((count & kValueMask) | kContinueBit); - *out++ = static_cast(((count >> 7) & kValueMask) | kContinueBit); - *out++ = static_cast(((count >> 14) & kValueMask) | kContinueBit); - *out++ = static_cast(((count >> 21) & kValueMask) | kContinueBit); - *out = static_cast(count >> 28); - } - } - - PROMPP_ALWAYS_INLINE void add_label(MarkedLabel label) noexcept { - const auto base_offset = this->buffer_.back().base_offset; - - if (label.name.is_reserved_name()) [[unlikely]] { - label.name.offset = 0; - label.name.length = 0; - } else { - label.name.offset -= base_offset; - } - label.value.offset -= base_offset; - - const uint8_t sz0 = encode_size(label.name.offset); - const uint8_t sz1 = encode_size(label.name.length); - const uint8_t sz2 = encode_size(label.value.offset); - const uint8_t sz3 = encode_size(label.value.length); - - const uint8_t layout = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); - - const uint32_t bytes_needed = sizeof(layout) + (sz0 + sz1 + sz2 + sz3) + 4; - const uint32_t offset = bytes_count(); - bytes_buffer_.resize(bytes_buffer_.size() + 17); - char* out = bytes_buffer_.data() + offset; - - *out++ = static_cast(layout); - - std::memcpy(out, &label.name.offset, sz0 + 1); - out += sz0 + 1; - std::memcpy(out, &label.name.length, sz1 + 1); - out += sz1 + 1; - std::memcpy(out, &label.value.offset, sz2 + 1); - out += sz2 + 1; - std::memcpy(out, &label.value.length, sz3 + 1); - - bytes_buffer_.resize(offset + bytes_needed); - } - - PROMPP_ALWAYS_INLINE void add_sample(const MarkedSample& sample) noexcept { - const double val = sample.sample.value(); - const bool has_ts = sample.has_ts; - - constexpr uint32_t max_sample_bytes = 1 + sizeof(sample.sample); // marker + value + timestamp - const uint32_t offset = bytes_count(); - bytes_buffer_.resize(offset + max_sample_bytes); - char* out = bytes_buffer_.data() + offset; - char* start = out; - - if (std::isnan(val)) [[unlikely]] { - *out++ = static_cast((has_ts ? 0b10000000 : 0) | 0b00000100); // NaN - } else if (val == 0.0) [[unlikely]] { - *out++ = static_cast((has_ts ? 0b10000000 : 0) | 0b00000000); // Zero - } else if (std::trunc(val) == val && val > 0.0) [[likely]] { - const uint64_t uval = static_cast(val); - if (uval <= std::numeric_limits::max()) { - const auto v = static_cast(uval); - out = write_marker_and_value(out, 0b00000001, has_ts, v); - } else if (uval <= std::numeric_limits::max()) { - const auto v = static_cast(uval); - out = write_marker_and_value(out, 0b00000010, has_ts, v); - } else if (uval <= std::numeric_limits::max()) { - const auto v = static_cast(uval); - out = write_marker_and_value(out, 0b00000011, has_ts, v); - } else { - out = write_marker_and_value(out, 0b00001001, has_ts, val); // double - } - } else { - float f = static_cast(val); - if (static_cast(f) == val) [[unlikely]] { - out = write_marker_and_value(out, 0b00001000, has_ts, f); // float - } else { - out = write_marker_and_value(out, 0b00001001, has_ts, val); // double - } - } - - if (has_ts) { - const auto ts = sample.sample.timestamp(); - std::memcpy(out, &ts, sizeof(ts)); - out += sizeof(ts); - } - - const uint32_t written = static_cast(out - start); - bytes_buffer_.resize(offset + written); - } - - private: - BareBones::Vector bytes_buffer_; - - template - PROMPP_ALWAYS_INLINE static char* write_marker_and_value(char* out, uint8_t marker, bool has_ts, const T& val) noexcept { - *out++ = static_cast((has_ts ? 0b10000000 : 0) | marker); - std::memcpy(out, &val, sizeof(T)); - return out + sizeof(T); - } - - PROMPP_ALWAYS_INLINE static uint8_t encode_size(uint32_t v) noexcept { - const uint32_t msb = (v == 0 ? 0 : 31 - std::countl_zero(v)); - return msb >> 3; - } - }; - - class MetadataMarkupBuffer : public MarkupBuffer { - public: - using Base = MarkupBuffer; - using Iterator = typename Base::Iterator; - using IteratorSentinel = typename Base::IteratorSentinel; - - [[nodiscard]] Iterator begin(std::string_view buffer) const noexcept { return {typename Base::Context{buffer}, this->buffer_.data(), this->items_count()}; } - [[nodiscard]] static IteratorSentinel end() noexcept { return {}; } - - PROMPP_ALWAYS_INLINE void add(MarkedString metric_name, MarkedString text, Prometheus::MetadataType type) noexcept { - this->buffer_.emplace_back(metric_name, text, type); - } - - void remove_item() noexcept { this->buffer_.pop_back(); } - void initialize(size_t reserve) noexcept { - this->buffer_.clear(); - this->buffer_.reserve(reserve); - } - }; - - [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_metadata() { + [[nodiscard]] Error parse_metadata() { static constexpr auto get_metadata_type = [](Token token) PROMPP_LAMBDA_INLINE { if (token == Token::kHelp) { return Prometheus::MetadataType::kHelp; @@ -639,13 +155,16 @@ class Scraper { return Error::kNoError; } - [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_metric() { + [[nodiscard]] Error parse_metric() { labels_.clear(); + marked_sample_ = {}; + marked_sample_.sample.timestamp() = default_timestamp_; + bool have_metric_name = false; auto& tokenizer = parser_.tokenizer(); - metric_buffer_.add_metric(tokenizer.token_str().data() - tokenizer.buffer().data()); + const uint32_t metric_offset = tokenizer.token_str().data() - tokenizer.buffer().data(); if (tokenizer.token() == Token::kMetricName) [[likely]] { labels_.push_back(MarkedLabel{.value = MarkedString::create(tokenizer.token_str(), tokenizer.buffer())}); @@ -670,39 +189,16 @@ class Scraper { return Error::kNoMetricName; } - // sort - { - const auto it = std::remove_if(labels_.begin(), labels_.end(), [](const MarkedLabel& label) { return label.value.is_empty(); }); - labels_.erase(it, labels_.end()); - } - - std::sort(labels_.begin(), labels_.end(), [buffer = tokenizer.buffer()](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { - return a.name.view(buffer) < b.name.view(buffer); - }); - - // hash - { - BareBones::XXHash3 hash; - for (const auto& label : labels_) { - hash.extend(label.name.view(tokenizer.buffer()), label.value.view(tokenizer.buffer())); - } - metric_buffer_.add_hash(hash.hash()); - } + const auto error = parse_metric_suffix(); - // encode count - { - metric_buffer_.add_count(labels_.size()); + if (error == Error::kNoError) [[likely]] { + encode_metric_data(metric_offset); } - // encode labels - for (const auto& label : labels_) { - metric_buffer_.add_label(label); - } - - return parse_metric_suffix(); + return error; } - [[nodiscard]] PROMPP_ALWAYS_INLINE Error tokenize_label_set(bool& have_metric_name) noexcept { + [[nodiscard]] Error tokenize_label_set(bool& have_metric_name) noexcept { auto& tokenizer = parser_.tokenizer(); tokenizer.next_non_whitespace(); @@ -744,7 +240,7 @@ class Scraper { return tokenizer.token() == Token::kBraceClose ? Error::kNoError : Error::kUnexpectedToken; } - [[nodiscard]] PROMPP_ALWAYS_INLINE Error get_label_name(MarkedString& label_name) const noexcept { + [[nodiscard]] Error get_label_name(MarkedString& label_name) const noexcept { auto& tokenizer = parser_.tokenizer(); if (tokenizer.token() == Token::kLabelName) [[likely]] { @@ -758,7 +254,7 @@ class Scraper { return Error::kUnexpectedToken; } - [[nodiscard]] PROMPP_ALWAYS_INLINE Error get_quoted_value(MarkedString& string) const noexcept { + [[nodiscard]] Error get_quoted_value(MarkedString& string) const noexcept { auto& tokenizer = parser_.tokenizer(); auto value = tokenizer.token_str(); @@ -782,33 +278,76 @@ class Scraper { return Error::kNoError; } - [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_metric_suffix() noexcept { + [[nodiscard]] Error parse_metric_suffix() noexcept { if (!parser_.is_value_token()) [[unlikely]] { return Error::kUnexpectedToken; } - MarkedSample sample{.sample = {default_timestamp_, {}}}; - if (const auto error = parse_sample(sample); error != Error::kNoError) { + if (const auto error = parse_sample(); error != Error::kNoError) { return error; } - metric_buffer_.add_sample(sample); return parser_.validate_parse_sample_result(); } - [[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_sample(MarkedSample& sample) noexcept { + [[nodiscard]] Error parse_sample() noexcept { auto& tokenizer = parser_.tokenizer(); - if (!parse_numeric_value(tokenizer.token_str(), sample.sample.value())) [[unlikely]] { + if (!parse_numeric_value(tokenizer.token_str(), marked_sample_.sample.value())) [[unlikely]] { return Error::kInvalidValue; } - if (std::isnan(sample.sample.value())) [[unlikely]] { - sample.sample.value() = Prometheus::kNormalNan; + if (std::isnan(marked_sample_.sample.value())) [[unlikely]] { + marked_sample_.sample.value() = Prometheus::kNormalNan; } tokenizer.next_non_whitespace(); - return parser_.parse_timestamp(sample.sample.timestamp(), sample.has_ts); + return parser_.parse_timestamp(marked_sample_.sample.timestamp(), marked_sample_.has_ts); + } + + void encode_metric_data(const uint32_t metric_offset) noexcept { + metric_buffer_.add_metric(metric_offset); + + sort_and_filter_labels(); + append_labels_hash(); + + metric_buffer_.bytes_enlarge(encoding::metric_maximum_encoding_size(labels_.size())); + + const encoding::LayoutMarker layout = + encoding::LayoutMarker::make(marked_sample_.has_ts, labels_.size(), encoding::SampleCodec::value_type(marked_sample_.sample.value())); + metric_buffer_.add_layout_and_count(layout, labels_.size()); + + encode_labels(metric_offset); + metric_buffer_.add_sample(layout, marked_sample_.sample); + } + + void encode_labels(const uint32_t offset) noexcept { + for (auto label : labels_) { + if (!label.name.is_reserved_name()) [[likely]] { + label.name.offset -= offset; + } + label.value.offset -= offset; + + metric_buffer_.add_label(label); + } + } + + void sort_and_filter_labels() noexcept { + const auto it = std::remove_if(labels_.begin(), labels_.end(), [](const MarkedLabel& label) PROMPP_LAMBDA_INLINE { return label.value.is_empty(); }); + labels_.erase(it, labels_.end()); + + std::sort(labels_.begin(), labels_.end(), [buffer = parser_.tokenizer().buffer()](const MarkedLabel& a, const MarkedLabel& b) PROMPP_LAMBDA_INLINE { + return a.name.view(buffer) < b.name.view(buffer); + }); + } + + void append_labels_hash() noexcept { + const auto& tokenizer = parser_.tokenizer(); + BareBones::XXHash3 hash; + for (const auto& label : labels_) { + hash.extend(label.name.view(tokenizer.buffer()), label.value.view(tokenizer.buffer())); + } + metric_buffer_.add_hash(hash.hash()); } Parser parser_; @@ -816,6 +355,7 @@ class Scraper { MetadataMarkupBuffer metadata_buffer_; BareBones::Vector labels_; Primitives::Timestamp default_timestamp_{}; + MarkedSample marked_sample_{}; }; using PrometheusScraper = Scraper; From f3e0c9469bbe1b127bd35e8a6f723fd83faeec51 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Wed, 13 May 2026 13:14:27 +0000 Subject: [PATCH 48/48] new final with opts --- pp/2026-04-29-scraper-opt-article-draft.md | 559 --------------------- pp/2026-04-29-scraper-opt-research.md | 413 --------------- pp/wal/benchmarks/scraper_benchmark.cpp | 2 - pp/wal/hashdex/scraper/encoding.h | 82 ++- pp/wal/hashdex/scraper/marked.h | 15 +- 5 files changed, 63 insertions(+), 1008 deletions(-) delete mode 100644 pp/2026-04-29-scraper-opt-article-draft.md delete mode 100644 pp/2026-04-29-scraper-opt-research.md diff --git a/pp/2026-04-29-scraper-opt-article-draft.md b/pp/2026-04-29-scraper-opt-article-draft.md deleted file mode 100644 index bd1a5577f8..0000000000 --- a/pp/2026-04-29-scraper-opt-article-draft.md +++ /dev/null @@ -1,559 +0,0 @@ -# Замедлить нельзя ускорить: как мы сократили память Scraper в Prom++ в 3.33× и не потеряли в скорости - -> Уровень сложности: сложный -> Время на прочтение: ~ 15–20 мин - -> [РИСУНОК-ОБЛОЖКА] — иллюстрация-метафора: «упаковали чемодан плотнее, но успели на самолёт». Можно сделать схему «байт-в-байт» — слева жирный layout, справа компактный. - -Всем привет! Меня зовут Глеб Шигин, я C++-разработчик в команде Deckhouse Prom++. Это статья про то, как мы прошли путь от «сделали наивный POC, выиграли в памяти, но просели в скорости» до «выиграли и в памяти, и в чтении» — на одном hot-path куске кода и за неделю последовательных коммитов. - -Кратко предыстория. Prom++ — наш форк Prometheus, в котором ядро хранения и обработки горячих данных переписано на C++, при сохранении полной совместимости с Prometheus и его periphery на Go (см. предыдущие статьи: «Deckhouse Prom++: мы добавили плюсы к Prometheus и сократили потребление памяти в 7,8 раза», «FastCGo»). После ряда оптимизаций storage, индексов и WAL, очередным «жирным» местом по памяти оказался **Scraper** — компонент, который превращает «сырой» текстовый дамп `/metrics` от target-а в наш markup-буфер. - -Что будет в статье: - -- Разберём, что такое Scraper в Prom++ и как у него устроен markup. -- Посмотрим на распределение реальных данных и сделаем дубовый POC, который ужимает память в 3.65×, но ломает чтение. -- Превратим POC в рабочий формат с тестами, заплатив за это просадкой по скорости. -- Отыграем скорость обратно — с помощью unrolled-декодера, `over-allocate-then-shrink` и LZCNT. -- На каждом этапе будем мерить Google Benchmark + смотреть Tracy-трейс. - -Содержание: - -- Контекст и мотивация -- Исходное решение -- Гипотеза и POC -- Превращаем POC в рабочий формат -- Отыгрываем скорость: read/write opt -- Дочищаем sample и parse_metric -- Итог -- Выводы - -> [ЗАМЕР-СТЕНД] Все бенчмарки проводились на одной машине: укажи реальную модель CPU, OS, версию clang/gcc, флаг `-c opt --copt=-march=native`, отсутствие ASan/TSan, отсутствие частотных скачков. Это блок «как у Владимира в FastCGo-статье»: nice -n -20, фиксированный CPU governor performance, и т.п. - -## TL;DR - -- Profiled Scraper, увидели, что: - - на каждую метрику тратится **28 + 16·N** байт на markup (N — число labels) + ~50% overshoot за счёт `reserve(buffer.size()/2)`; - - реальные `name.length`, `value.length`, относительные offset-ы почти всегда умещаются в 1–2 байта; - - >95% значений — маленькие положительные целые counters, timestamp в большинстве случаев отсутствует. -- Сделали variable-length encoding для labels и samples + относительные offset'ы + отдельный header-массив `Vector` поверх variable-tail `Vector`. -- POC просел по скорости, но мы вернули её обратно за счёт unrolled-загрузок переменной длины, `resize+write+shrink` и `countl_zero` для расчёта длины. -- Итог: память **−70% (×3.33)**, read **+33..42%**, parse — небольшой проигрыш в худшем кейсе. - -> [ЗАМЕР-ИТОГ] График / таблица из 4 столбцов: parse, read, allocated_memory, ось X — ревизия. Две панели: cehp-like и m3-like дампы. Это ровно тот самый итоговый сводный график, на который дальше ссылаемся. - -## Контекст и мотивация - -Scraper — компонент, который раз в `scrape_interval` секунд получает HTTP-ответ от target-а в формате Prometheus text exposition (или OpenMetrics) и распарсивает его в нашу внутреннюю модель. Дальше его данные забирают шарды и складывают в head/WAL. Между «парсингом» и «забором шардом» данные живут в **markup-буфере** Scraper-а. - -``` -target /metrics ──HTTP──▶ PrometheusParser ──tokenize──▶ Scraper::parse ──▶ markup buffer - │ - ├── shard 0 ── metric.read(ts) - ├── shard 1 ── metric.read(ts) - └── ... -``` - -> [РИСУНОК-1] Аккуратная схема пайплайна. Хорошо бы показать, что markup-буфер живёт целиком, пока все шарды не вычитают свою половину/треть. - -Сколько мы сейчас платим за этот буфер? Берём типичный дамп `kube-apiserver /metrics` (~13 МБ текста, ~85 тысяч метрик) и пишем простой бенчмарк: - -```cpp -void ScraperParse(benchmark::State& state) { - ZoneScoped; - const auto str = get_file_content(); - std::string tmp_str; - tmp_str.resize(str.size()); - - for ([[maybe_unused]] auto _ : state) { - std::memcpy(tmp_str.data(), str.data(), str.size()); - PrometheusScraper scraper; - std::ignore = scraper.parse(tmp_str, 0); - } - - PrometheusScraper scraper; - auto tmp_str2 = str; - std::ignore = scraper.parse(tmp_str2, 0); - state.counters["Alloc"] = - benchmark::Counter(static_cast(scraper.allocated_memory()), - benchmark::Counter::kDefaults, - benchmark::Counter::OneK::kIs1024); -} -``` - -> [ЗАМЕР-BASELINE] Стартовые числа на baseline-коммите `3f0b2bd72`: -> -> - cehp-like дамп: ScraperParse ≈ 38 ms, ScraperRead ≈ 11 ms, Alloc ≈ **13.7 MiB**. -> - m3-like дамп: ScraperParse ≈ 23 ms, ScraperRead ≈ 1.8 ms, Alloc ≈ **13.7 MiB**. - -Чтобы было удобно, сразу зафиксируем три бенчмарка: - -- `Parser` — голая токенизация. Не KPI, нужна только чтобы вычитать «тело» из `ScraperParse`. -- `ScraperParse` — полный `Scraper::parse`. KPI по записи + `Alloc`. -- `ScraperRead` — полный проход `for (auto& m : metrics) m.read(ts)`. KPI по чтению. - -«Чистое время скрейпера» дальше будет фигурировать как `Plain Scraper Time = ScraperParse - Parser`. Это позволяет не путать прирост на токенизаторе с приростом на самом скрейпере. - -## Исходное решение - -Внутри Scraper лежит `MetricMarkupBuffer` — обычный `BareBones::Vector` (наша обёртка над контейнером с управляемой памятью). В него подряд складываются метрики: - -```cpp -#pragma pack(push, 1) -struct MarkedString { uint32_t offset; uint32_t length; }; // 8 B -struct MarkedLabel { MarkedString name; MarkedString value; }; // 16 B - -struct MarkedLabelSet { - uint32_t count; - MarkedLabel labels[]; -}; - -struct MarkedMetric { - uint64_t hash; // 8 - Primitives::Sample sample; // 16 (double value + int64 timestamp) - MarkedLabelSet label_set; // 4 + 16 * N -}; -#pragma pack(pop) -``` - -Итого на одну метрику: **28 + 16 · N** байт. Плюс при старте парсинга буфер резервируется как `metric_buffer_.initialize(buffer.size() / 2)`, что для 13 МБ дампа сразу даёт ~6.5 МБ и ещё 3.25 МБ на metadata. - -> [РИСУНОК-LAYOUT-V0] Картинка байт-в-байт: один `MarkedMetric` с тремя `MarkedLabel` подряд, цветом обозначить, что хранится. Подсветить, что на `__name__` тоже тратится 16 байт. - -На что мы посмотрели в первую очередь: - -> [ЗАМЕР-РАСПРЕДЕЛЕНИЕ] Гистограммы по реальному дампу: -> -> - длины `name`/`value` (доля укладывающихся в 1, 2, 3, 4 байта) — ожидаем, что 1–2 байта покрывают 95%+; -> - распределение типов значений: zero / uint8 / uint16 / uint32 / float / double / NaN — ожидаем доминирование small uint; -> - доля метрик с собственным timestamp — ожидаем, что подавляющее большинство без него. - -Эти три картинки и есть «топливо» для дальнейших гипотез. В нашей нагрузке: - -- 95+% `name.length` ≤ 255, 99+% `value.length` ≤ 65535; -- ~99% значений — целые положительные counters; -- timestamp в самой строке — единичные случаи (например, `pushgateway`). - -То есть мы платим за worst-case, которого почти никогда не бывает. - -## Гипотеза и POC - -«Если данных мало, давайте кодировать переменной длиной». Проблема в том, что переменная длина — это либо varint (с continue-битами), либо префиксы-длины. И то и другое — лишние ветки в hot loop. Поэтому первая попытка — два независимых POC: один — переменная упаковка labels, другой — переменная упаковка sample. Их **специально** делали без read-пути и без тестов, чтобы быстро увидеть «потолок» по памяти. - -### POC 1 — labels varint - -Идея простая. Каждый `MarkedString` хранит `offset` и `length`, оба — `uint32`. Вместо этого: - -- `offset`-ы делаем **относительными** к началу самой метрики. Тогда они почти всегда влезают в 1 байт. -- Каждое из 4 полей label-а (`name.offset`, `name.length`, `value.offset`, `value.length`) кодируем 1, 2, 3 или 4 байтами в зависимости от величины. -- В начале label-а кладём 1-байтовый **layout descriptor**, в котором по 2 бита на каждое из 4 полей описывают, сколько байт оно занимает. -- `__name__` — это специальный label, который встречается ровно один раз на метрику. Его имя не нужно хранить вовсе, обозначаем «(0, 0)» в полях имени и при чтении подставляем константу `Prometheus::kMetricLabelName`. -- Количество labels тоже varint: 1–5 байт. - -```cpp -const uint8_t sz0 = encode_size(label.name.offset); -const uint8_t sz1 = encode_size(label.name.length); -const uint8_t sz2 = encode_size(label.value.offset); -const uint8_t sz3 = encode_size(label.value.length); - -const uint8_t layout = (sz0) | (sz1 << 2) | (sz2 << 4) | (sz3 << 6); - -std::array tmp{}; -char* out = tmp.data(); -*out++ = static_cast(layout); -*reinterpret_cast(out) = label.name.offset; out += sz0 + 1; -*reinterpret_cast(out) = label.name.length; out += sz1 + 1; -*reinterpret_cast(out) = label.value.offset; out += sz2 + 1; -*reinterpret_cast(out) = label.value.length; out += sz3 + 1; - -this->buffer_.push_back(tmp.data(), out); -``` - -Best case: **5 байт/label** против 16. Worst: 17. - -> [РИСУНОК-LABEL-LAYOUT] Картинка одного label-а: 1 байт layout + 4 переменных поля, цветом подсветить, какое поле сколько байт. На втором кадре — то же самое для `__name__` (ноль байт на name). - -Чтобы лейблы можно было упаковать в правильном порядке (с одинаковым хешем по сравнению с baseline), приходится сначала собрать их в скретч-вектор `BareBones::Vector labels_` (один на класс, `reserve(255)`), отсортировать по name, посчитать xxhash, и только потом батчем эмитить в основной буфер. Это +1 проход по labels на каждую метрику. - -### POC 2 — sample encoding - -Идея ещё проще. `Sample` — это всегда 16 байт (8 на double, 8 на timestamp). При том, что почти всегда: - -- timestamp равен default-у (т.е. его нет в дампе), -- value — небольшое положительное целое. - -Кодируем 1 байтом маркера + переменное тело: - -``` -bit 7 : has_ts (если 1 — за value идут 8 байт timestamp) -bits 0..3 : тип значения - 0000 zero (0 байт) - 0001 uint8 (1 байт) - 0010 uint16 (2 байта) - 0011 uint32 (4 байта) - 0100 staleNaN (0 байт) - 1000 float32 (4 байта) - 1001 double (8 байт) -``` - -Best case: **1 байт** на sample (нулевой counter без TS); worst — 17 байт (double + ts), почти не хуже фиксированных 16. - -```cpp -if (std::isnan(val)) [[unlikely]] { flush(0b00000100); return; } // NaN -if (val == 0.0) [[unlikely]] { flush(0b00000000); return; } // zero -if (std::trunc(val) == val && val > 0.0) [[likely]] { - auto ival = static_cast(val); - if (ival <= 0xFF) { flush(0b00000001); append((uint8_t)ival); return; } - if (ival <= 0xFFFF) { flush(0b00000010); append((uint16_t)ival); return; } - if (ival <= 0xFFFFFFFFULL) [[likely]] { flush(0b00000011); append((uint32_t)ival); return; } -} -float f = static_cast(val); -if (static_cast(f) == val) [[unlikely]] { flush(0b00001000); append(f); return; } // float32 -flush(0b00001001); append(val); // double -``` - -> [РИСУНОК-SAMPLE-LAYOUT] Маркер-байт + варианты тел. Хорошо смотрится «дерево» из 7 веток с подписанными байтами на каждом листе. - -### Что показал POC - -> [ЗАМЕР-POC] Прогон только `ScraperParse` + `Alloc` (read-пути ещё нет, поэтому соответствующие колонки пустые). Стиль — как у Владимира в FastCGo с benchstat-выводом. -> -> | Ревизия | Parse, ms (×) | Alloc, MiB (×) | Read | -> |---|---:|---:|---| -> | baseline `3f0b2bd72` | 38.2 (1.00) | 13.7 (1.00) | 11 ms | -> | + labels varint | 45.8 (0.83) | 5.0 (×2.74) | n/a | -> | + sample encoding | 47.8 (0.80) | 3.8 (×3.65) | n/a | -> | оба POC вместе (`4656f7137`) | 47.5 (0.80) | 4.15 (×3.31) | сломан | - -Итог POC: память жёстко сэкономили (×3.65), но: - -1. за это заплатили ~20% времени `parse` — добавились ветвистая запись label-а, цикл по 4 разрядностям, цикл по marker-типам; -2. сломали read-путь — он не реализован под новый формат, а старый perevich `Sample`/`MarkedLabel` уже не лежит в буфере; -3. тесты, естественно, красные. - -Это нормально и даже полезно: мы увидели **потолок** по памяти. Дальше задача — вернуть скорость и чтение, не отдав обратно эти 9.5 МБ. - -## Превращаем POC в рабочий формат - -Первое, что мы сделали — реализовали полный read-декодер и сделали тесты зелёными. Параллельно встретились с архитектурным неудобством: чтобы итерироваться по такому variable-length буферу, нужно либо знать длину каждого записанного элемента (т.е. ещё раз декодировать заголовок), либо хранить длину рядом. - -Мы выбрали третий путь — **разнесли header и tail по разным контейнерам**: - -```cpp -class MetricMarkupBuffer { - ... - BareBones::Vector metric_buffer_; // фиксированный header per metric (16 B) - BareBones::Vector bytes_buffer_; // variable tail: count + labels + sample -}; - -struct MarkedMetric { - uint64_t hash; // 8 - uint32_t base_offset; // 4 — начало метрики в исходном буфере (для относительных offset-ов) - uint32_t data_offset; // 4 — начало tail в bytes_buffer_ -}; -``` - -> [РИСУНОК-LAYOUT-V1] Двухэтажная схема: сверху `Vector` — все «шапки» подряд, снизу `Vector` — все упакованные «хвосты» подряд. Стрелочка `data_offset` от шапки к хвосту. - -Что это даёт: - -- `++iterator` теперь — это `++ptr_` по плотному `MarkedMetric*`. Не нужно декодировать tail только чтобы посчитать длину. -- На `metric_buffer_` (16 B/запись) шарды ходят кеш-friendly при чисто «hash-обходах». -- `Sample` целиком переехал в variable tail; `default_timestamp` поднялся на уровень всего `Scraper`-а — один на дамп. Это ещё минус 8 байт «обычной» метрики. - -Read-декодер на эту схему получается прямолинейно: - -```cpp -template -void read(Timeseries& ts) const { - const char* ptr = bytes_buffer_.data() + item_->data_offset; - const char* base = buffer_.data() + item_->base_offset; - - uint32_t labels_count = decode_varint(ptr); - ts.label_set().reserve(labels_count); - - for (uint32_t i = 0; i < labels_count; ++i) { - const uint8_t layout = static_cast(*ptr++); - const uint8_t sz0 = (layout >> 0) & 0x3; - const uint8_t sz1 = (layout >> 2) & 0x3; - const uint8_t sz2 = (layout >> 4) & 0x3; - const uint8_t sz3 = (layout >> 6) & 0x3; - - auto read_val = [&](uint8_t sz) PROMPP_LAMBDA_INLINE -> uint32_t { - uint32_t v = 0; - memcpy(&v, ptr, sz + 1); // <-- внимание на эту строку - ptr += sz + 1; - return v; - }; - - uint32_t name_off = read_val(sz0); - uint32_t name_len = read_val(sz1); - uint32_t value_off = read_val(sz2); - uint32_t value_len = read_val(sz3); - - // ... append ... - } - // ... sample ... -} -``` - -Тесты зелёные. Сериализация и десериализация совместимы. Smoke-pass идёт. - -> [ЗАМЕР-READ-WORKS] Та же таблица, но добавились колонки `Pass` и `Read`: -> -> | Ревизия | Parse (×) | Pass (×) | Read (×) | Alloc (×) | -> |---|---:|---:|---:|---:| -> | baseline | 1.00 | 1.00 | 1.00 | 1.00 | -> | `4656f7137` initial encoding | 0.80 | n/a | broken | 3.31 | -> | `0027219f0` read works | 0.76 | 0.70 | **0.65** | 3.33 | - -Read стал **медленнее** baseline. Не на единицы процентов — на 35%. - -> «Просчитался, но где?!» - -Заглядываем в Tracy (включаем `--//bazel/toolchain:profiling=1`, в `Metric::read` и `parse_metric` выставляем `ZoneScopedN` на под-фазы — `decode_count`, `decode_labels`, `decode_sample`): - -> [РИСУНОК-TRACY-READ-V1] Скриншот Tracy для `Metric::read` на `0027219f0`. Видно, что `decode_labels` занимает абсолютное большинство времени, а внутри него — узкое место в строке `memcpy(&v, ptr, sz + 1)`. - -Дело в той самой строчке `memcpy(&v, ptr, sz + 1)` с **переменным** размером. Компилятор не разворачивает её в простой load — он эмитит `memcpy`-fallback (или generic loop). Каждый label-байт стоит отдельной инструкции call/branch. Аналогично `add_label` на write-пути сначала пишет в `std::array tmp`, потом делает `push_back(tmp, len)` — лишний копирующий проход. - -## Отыгрываем скорость: read/write opt - -Это самый важный коммит во всей серии — `9b62e6ae`. Здесь мы возвращаем скорость, не теряя ни байта памяти. - -### Read: разворачиваем «memcpy с переменной длиной» руками - -```cpp -PROMPP_ALWAYS_INLINE -static uint32_t read_val_partial(const char*& p, uint8_t sz) noexcept { - if (sz == 0) [[likely]] { - const uint32_t v = static_cast(p[0]); - p += 1; - return v; - } - if (sz == 1) { - const uint32_t v = static_cast(static_cast(p[0])) - | (static_cast(static_cast(p[1])) << 8); - p += 2; - return v; - } - if (sz == 2) { - const uint32_t v = static_cast(static_cast(p[0])) - | (static_cast(static_cast(p[1])) << 8) - | (static_cast(static_cast(p[2])) << 16); - p += 3; - return v; - } - uint32_t v; - std::memcpy(&v, p, 4); - p += 4; - return v; -} -``` - -После этого декодирование одного label-а становится цепочкой прямых нагружений + сдвигов. Switch по типу sample заодно отсортировали по частоте: `uint8 / zero / uint16 / uint32 / staleNaN / float / double`. `decode_varint` и `read_val_partial` пометили `PROMPP_ALWAYS_INLINE` — без этого компилятор для коротких функций делал отдельные `call`. - -### Write: `over-allocate-then-shrink` - -Та же идея, но в зеркале. Раньше `add_label` писал в локальный `std::array tmp` и делал `push_back(tmp, used_size)`. Это два прохода. - -Стало: - -```cpp -const uint32_t bytes_needed = sizeof(layout) + (sz0 + sz1 + sz2 + sz3) + 4; -const uint32_t offset = bytes_count(); -bytes_buffer_.resize(bytes_buffer_.size() + 17); // worst-case заранее -char* out = bytes_buffer_.data() + offset; - -*out++ = static_cast(layout); -std::memcpy(out, &label.name.offset, sz0 + 1); out += sz0 + 1; -std::memcpy(out, &label.name.length, sz1 + 1); out += sz1 + 1; -std::memcpy(out, &label.value.offset, sz2 + 1); out += sz2 + 1; -std::memcpy(out, &label.value.length, sz3 + 1); - -bytes_buffer_.resize(offset + bytes_needed); // shrink to actual -``` - -Что мы здесь выиграли: - -- В `bytes_buffer_` пишем **сразу**, без промежуточного буфера. -- `resize(+17)` обеспечивает «есть куда писать»; внутри `memcpy(out, &val, sz+1)` нет проверок границ. -- В конце `resize` сжимает до фактически записанного размера. У `BareBones::Vector` это просто корректировка `size_`, без пересборки. - -Тот же приём применили к `add_count` (varint длиной до 5 байт). - -### Write: `encode_size` через LZCNT - -Раньше: - -```cpp -static uint8_t encode_size(uint32_t v) noexcept { - if (v <= 0xFF) return 0; - if (v <= 0xFFFF) return 1; - if (v <= 0xFFFFFF) return 2; - return 3; -} -``` - -Стало: - -```cpp -PROMPP_ALWAYS_INLINE static uint8_t encode_size(uint32_t v) noexcept { - const uint32_t msb = (v == 0 ? 0 : 31 - std::countl_zero(v)); - return msb >> 3; -} -``` - -Каскад из 3 веток заменили на одну инструкцию `lzcnt`/`bsr` + сдвиг. На hot-path этот вызов идёт ровно 4 раза на каждый label, поэтому экономия чувствуется. - -### Что это дало - -> [ЗАМЕР-RW-OPT] Прогон 3 бенчмарков: -> -> | Ревизия | Parse (×) | Pass (×) | Read (×) | Alloc (×) | -> |---|---:|---:|---:|---:| -> | `0027219f0` read works | 0.76 | 0.70 | 0.65 | 3.33 | -> | `9b62e6ae` r/w opt | **0.83** | **0.79** | **1.39** (cehp) / **1.39** (m3) | 3.33 | - -Read **+114%** на cehp-like дампе и **+289%** на m3-like (за счёт более коротких labels). Pass прибавил ~12%. Память — ровно на месте. - -> [РИСУНОК-TRACY-READ-V2] Тот же скриншот Tracy `Metric::read`, но после `9b62e6ae`. `decode_labels` ужалось примерно в 2 раза. - -## Дочищаем sample и parse_metric - -После главного «win-back» осталось два аккуратных шага. - -### `aca48508` — тот же приём для samples - -`add_sample` страдал от той же болезни, что и `add_label` до `9b62e6ae`: внутренний `flush()` лямбдой, отдельные `push_back` для маркера, для значения, для timestamp. Применяем ровно то же — `resize(+17)`, последовательный курсор, в конце `resize(actual)`: - -```cpp -constexpr uint32_t max_sample_bytes = 1 + sizeof(sample.sample); // 1 marker + 8 value + 8 ts -bytes_buffer_.resize(offset + max_sample_bytes); -char* out = bytes_buffer_.data() + offset; -char* start = out; - -if (std::isnan(val)) [[unlikely]] { - *out++ = static_cast((has_ts ? 0b10000000 : 0) | 0b00000100); -} else if (val == 0.0) [[unlikely]] { - *out++ = static_cast((has_ts ? 0b10000000 : 0) | 0b00000000); -} else if (std::trunc(val) == val && val > 0.0) [[likely]] { - // ... uint8 / uint16 / uint32 / fallback double ... -} else { - // ... float32 / double ... -} - -if (has_ts) { - std::memcpy(out, &sample.sample.timestamp(), 8); - out += 8; -} - -bytes_buffer_.resize(offset + (out - start)); -``` - -Параллельно поменяли layout: timestamp **после** value (раньше — до). На декодер влияет минимально, но позволяет в записи использовать один общий курсор без переустановок. - -> [ЗАМЕР-SAMPLE-OPT] Очень небольшая прибавка в pass (cehp 34.3 → 33.6 ms, m3 16.9 → 16.3 ms), read — почти шум. Но это «бесплатное» улучшение в стиле уже сделанного. - -### `2036d7ea` — `MetricParser` → метод - -В старой схеме на каждую метрику стек-локально создавался объект: - -```cpp -if (const auto error = MetricParser{parser_, metric_buffer_, labels_, - static_cast(tokenizer.token_str().data() - tokenizer.buffer().data()), - default_timestamp}.parse(); - error != Error::kNoError) [[unlikely]] { - metric_buffer_.remove_item(); - return error; -} -``` - -`MetricParser` — это вложенный класс с 4 ссылками-членами. Компилятор имеет полное право его не инлайнить — каждая метрика стоит лишнего конструктора + indirection через ref-поля. - -Превратили в обычный метод `Scraper`: - -```cpp -[[nodiscard]] PROMPP_ALWAYS_INLINE Error parse_metric() { - ... -} -``` - -Все «ссылки» теперь — обычные `this->labels_`, `this->metric_buffer_`. `PROMPP_ALWAYS_INLINE` теперь работает по-настоящему. - -> [ЗАМЕР-INLINE] Pass cehp 34.0 → 33.2 ms, m3 17.4 → 16.3 ms. Прибавка маленькая, но стабильная и видна на каждом прогоне. На Tracy исчез отдельный фрейм конструктора `MetricParser`. - -> [РИСУНОК-TRACY-INLINE] Сравнение двух Tracy-кадров: до и после, видно отсутствие лишнего фрейма. - -Дальше (`1113d04b`) — чистая косметика: вынес `process_labels_buffer`, `sort_and_filter_labels`, `append_labels_hash` в отдельные методы. На скорости/памяти не сказывается, но код становится читаемым. - -## Итог - -> [ТАБЛИЦА-ИТОГ] Сводная таблица из исследования. Структура: ревизия → ключевое изменение → Parse, Pass, Read, Alloc. Можно вставить из `2026-04-29-scraper-opt-research.md` секция 7 — её достаточно «причесать». - -Чтобы было нагляднее, итог одной картинкой: - -> [РИСУНОК-ИТОГ] 4-панельный график (или 2×2): по оси X — ревизии в хронологическом порядке, по Y — нормированное к baseline время/память. Панели: -> -> 1. Plain Scraper Time (cehp) -> 2. Plain Scraper Time (m3) -> 3. Read time (cehp/m3 на одной картинке) -> 4. Allocated memory -> -> Пунктирной линией — baseline = 1.0. Видны три «горки»: подъём `4656f7137` / `0027219f0`, спуск `9b62e6ae`, плоский конец. - -Финальный layout одной метрики: - -> [РИСУНОК-LAYOUT-FINAL] Сверху — массив `MarkedMetric` (16 B на запись). Снизу — variable tail в `bytes_buffer_`: varint count → последовательность labels (1 байт layout + 4..16 байт полей) → 1 байт marker + value + опц. timestamp. - -Финальные числа: - -- На наших дампах **markup-память ужалась в 3.33×** (с 13.7 МБ до 4.13 МБ). -- На «толстом» дампе read стал на **33% быстрее**, на плотном — на **42% быстрее**. -- На write — небольшой проигрыш (от −5% до −20% относительно baseline в зависимости от дампа). В нашем pipeline write делается один раз на интервал scrape-а, read — N раз на N шардов, поэтому сделка для нас — отличная. - -## Выводы - -- **Domain-specific knowledge — главный инструмент**. Все приёмы в этой статье — `varint`, layout-byte, `__name__`-elision, `default_timestamp` per-scraper — обоснованы тем, **как реально выглядят** Prometheus-дампы. Без гистограмм по реальным данным мы бы не выбрали 4-разрядное layout-кодирование (а взяли бы скучный `LEB128`/`varint`, и проиграли бы по скорости из-за continue-битов). -- **`memcpy` с переменной длиной — не SIMD**. Если длина — известный compile-time-набор из 1..4 байт, разверните руками. Это вернуло нам ~2× на read. -- **Over-allocate-then-shrink**. Если максимальная длина записи известна и невелика, проще сделать `resize(+max)`, писать линейно без проверок границ, а в конце `resize(actual)`. У нас это вторая половина выигрыша. -- **Считайте байты вычислениями**. `(31 - countl_zero(v)) >> 3` короче и быстрее каскада `if`, и читается ничем не хуже. -- **Inline-friendly код**. Вложенные классы с ref-членами на hot-path — ловушка. Если функция вызывается на каждой метрике — лучше метод того же класса с `ALWAYS_INLINE`. -- **Профилируйте на каждом шаге**. Tracy + Google Benchmark показывают разницу между «кажется, должно быть быстрее» и «вот тут стоит ровно тот memcpy». В нашем случае POC, который выглядел как «победа», на read-пути оказался регрессом — и без трейсов мы бы это пропустили. - -> [ЗАМЕР-FINAL-SUMMARY] Здесь хорошо смотрится один итоговый бар-чарт: «было — стало» по 4 метрикам, желательно на двух типах нагрузки. - -## Что осталось за кадром - -В этой статье мы не разбираем: - -- что происходит с `metadata_buffer_` (`# HELP / # TYPE / # UNIT`) — там оптимизаций пока не делали; -- как именно `BareBones::Vector` обрабатывает `resize(+N)` без полной реаллокации; -- интеграцию Scraper-а с шардами (это отдельная история про `hash() % N`-распределение). - -Если интересно про что-то из этого — пишите в комментариях. - -## P. S. - -Читайте также в нашем блоге: - -- Deckhouse Prom++: мы добавили плюсы к Prometheus и сократили потребление памяти в 7,8 раза -- FastCGo: как мы ускорили вызов C-кода в Go в 16,5 раза -- (другие наши статьи по Prom++) - -Теги: `prom++`, `prometheus`, `c++`, `optimization`, `bit packing`, `varint`, `tracy`, `google benchmark` - ---- - -## Чек-лист правок и допросов перед публикацией - -> Это служебный блок для нас, в финальный текст не идёт. - -- [ ] Проверить, что числа в TL;DR сходятся с финальным графиком. -- [ ] Подтвердить термины: «scraper» / «скрейпер» — выбрать одно написание и придерживаться. -- [ ] Все имена коммитов оставить как есть (в стиле автора-оригинала) либо переименовать в человекопонятные (например, `4656f7137` → «POC: initial encoding»). -- [ ] Решить, показываем ли мы фактические Tracy-скриншоты или схематично рисуем «как было / как стало». -- [ ] Снять все три гистограммы распределения с реальных дампов. -- [ ] Перепрогнать все замеры на одной чистой машине, единым прогоном через `bench_sweep.sh`. -- [ ] Свести итоговый график. -- [ ] Указать ссылку на репозиторий с примерами / на сам Prom++. diff --git a/pp/2026-04-29-scraper-opt-research.md b/pp/2026-04-29-scraper-opt-research.md deleted file mode 100644 index 826353c1eb..0000000000 --- a/pp/2026-04-29-scraper-opt-research.md +++ /dev/null @@ -1,413 +0,0 @@ -# Scraper optimization — research dump - -Сводка по серии коммитов на ветке `scraper-rw-opt`, в которой `Scraper` -в `pp/wal/hashdex/scraper/scraper.h` был оптимизирован по памяти и -скорости. Используется как рабочий black-book перед написанием статьи -на Хабр. - -Период: `3f0b2bd72` (2025-08-25) → `1113d04b` (2025-09-01). - -## 1. Что измеряется - -Бенчмарки лежат в `pp/wal/benchmarks/scraper_benchmark.cpp`. - -- `BenchmarkParser` — голая токенизация (`PrometheusParser::tokenizer().tokenize` + `next()` в цикле). Это **не KPI**, а вычитаемая база, чтобы получить «чистое» время скрейпера: `Plain Scraper Time = ScraperParse - Parser`. -- `BenchmarkScraperParse` — полный `Scraper::parse(buffer, 0)` + замер `allocated_memory()`. -- `BenchmarkScraperRead` — после одного `parse()` итерируем по `scraper.metrics()` и каждому `metric.read(ts)` заполняем `TimeseriesSemiview`. - -Две колонки в твоей CSV: - -- `cehp-runner` — более «толстый» промовский дамп (длинные labels, смешанные значения; `parse ≈ 38..50 ms`, alloc baseline ≈ 13.7 MiB). -- `m3` — более плотный дамп (короткие labels, в основном целые counters; `parse ≈ 23..30 ms`, alloc такой же). - -Файлы фикстур у тебя локальные. В `scraper.flags` указан путь -`…/wal/benchmarks/data/blobs.txt`, в репо его нет. Сейчас в `/prompp/pp/` -лежит только один дамп — `kube-api.metrics` (~13M, 85k строк, стиль -ближе к cehp-runner). - -## 2. Базовый layout (`3f0b2bd72`) - -```cpp -struct MarkedString { uint32 offset; uint32 length; }; // 8 B -struct MarkedLabel { MarkedString name; MarkedString value; }; // 16 B -struct MarkedLabelSet { uint32 count; MarkedLabel labels[]; }; -struct MarkedMetric { - uint64 hash; // 8 - Sample sample; // 16 (double + int64) - MarkedLabelSet ls; // 4 + 16*N -}; -``` - -Итого на одну метрику: **28 + 16·N байт** + `metric_buffer_.initialize(buffer.size()/2)` overshoot. - -Поверх — `MetricParser{...}.parse()` инстанцируется на каждую метрику, -читает токены, по ходу вызывает `markup_buffer_.add_label(...)`, в -конце — `calculate_hash()` (sort + xxhash). - -Сам коммит `3f0b2bd72` — это разделение одного `BenchmarkScraper` на -`Parse`/`Read` + добавление счётчика `Alloc`. Это и есть baseline. - -«Жирное» в этом виде: -1. На `__name__` тратится полный `MarkedLabel` 16 B — у каждой метрики. -2. Большинство `offset`/`length` в реальной нагрузке умещаются в 1–2 байта, а резервируются 4. -3. `Sample` всегда 16 B, хотя ~99% значений в Prom-text — маленькие положительные целые counters/gauges. -4. `metric_buffer_.initialize(buffer.size()/2)` берёт «с запасом», что и даёт 13.7 MiB. - -## 3. Хронология коммитов - -| # | Hash | Тема | -|---|------|------| -| 0 | `3f0b2bd72` | benchmark update (baseline) | -| POC₁ | (не закоммичено) | **labels varint** — POC переменной упаковки label-полей | -| POC₂ | (не закоммичено) | **sample encoding** — POC переменной упаковки value/timestamp | -| 1 | `4656f7137` | initial encoding — оба POC сведены воедино, тесты ещё не проходят, read «болт» | -| 2 | `0027219f0` | read + passing all tests — реализован декодер, разбили хранилище на `metric_buffer_` (header) + `bytes_buffer_` (variable tail) | -| 3 | `9b62e6ae` | read/write optimization — главный прирост по скорости | -| 4 | `aca48508` | sample optimization — тот же приём для samples | -| 5 | `26d0af47` | benchmark only: read лишь чётных hash (имитация sharding) | -| 6 | `2036d7ea` | `MetricParser` → метод `parse_metric()` (inline-friendly) | -| 7 | `1113d04b` | extract-method косметика над `parse_metric` | - -POC-шаги `labels varint` и `sample encoding` в git-истории отдельно -не сохранились — их собрали в коммит `4656f7137` («initial encoding»). -Сами замеры по ним есть в CSV: только `parse, ns` и `alloc, Mi`, -без `pass` и без `read`. Это согласуется с тем, что: - -- read-декодера на тот момент ещё не было (в `MarkedMetric` стоял placeholder `uint8_t bytes[]` без реального сериализатора); -- `Plain Scraper Time` зависит от рабочего read-пути, который тоже ещё не считался. - -То есть POC-числа — это ровно те моменты, когда уже сжимали запись и -хотели увидеть, **на сколько падает alloc** и **сколько за это платим -в parse**, не задумываясь про корректность чтения. - -## 4. Что и зачем в каждом коммите - -### 4.1. POC₁ «labels varint» (часть `4656f7137`) - -**Гипотеза.** На реальном Prom-тексте `offset`/`length` для name и value -почти всегда умещаются в 1 байт; offset метрики тоже невелик, если -хранить его относительно начала самой метрики; `__name__` встречается -ровно один раз на метрику и его имя не нужно хранить вовсе. - -**Что сделано.** -- Дополнительное поле в `MarkedMetric`: `offset` — глобальное смещение «первого» токена метрики в исходном буфере. Все остальные `MarkedString::offset` хранятся относительно него. -- Каждая `MarkedLabel` сериализуется как: - - 1 байт **layout descriptor**: `(sz0)|(sz1<<2)|(sz2<<4)|(sz3<<6)`, где каждое `szk ∈ [0..3]` означает «занимает `szk+1` байтов». - - далее 4 поля `name.offset`, `name.length`, `value.offset`, `value.length`, по 1..4 байта каждое. - - best case: **5 байт/label** (vs 16); worst case: 17. -- `__name__` детектируется через `is_reserved_name()` → name.offset/length = 0/0 (декодер потом восстановит). -- Количество label'ов — **varint 1..5 байт** (вместо 4 фиксированных). -- Лейблы сначала собираются в скретч-вектор `labels_` (`reserve(255)` на класс), сортируются по name, по ним считается xxhash, и только потом батчем пишутся в основной буфер. -- Снят `metric_buffer_.initialize(buffer.size()/2)` overshoot. Буфер растёт по ходу. - -**Цена.** Ветвистая по 4 разрядностям запись + копирование labels через скретч + сортировка. По CSV: parse 38.2 → 45.8 ms (cehp), 22.8 → 28.8 ms (m3). Зато **alloc 13.7 → 5.0 MiB (×2.74)**. - -### 4.2. POC₂ «sample encoding» (часть `4656f7137`) - -**Гипотеза.** `Sample` 16 B всегда — расточительно: TS обычно -отсутствует (используется default), value — почти всегда маленький -положительный uint, реже — float, изредка — NaN/staleNaN. - -**Что сделано.** Один маркер-байт: - -``` -bit 7 : has_ts (если 1 — за value идут 8 байт timestamp) -bits 0..3 : тип значения - 0000 zero (0 байт) - 0001 uint8 (1 байт) - 0010 uint16 (2 байта) - 0011 uint32 (4 байта) - 0100 staleNaN (0 байт) - 1000 float32 (4 байта) - 1001 double (8 байт) -``` - -Best case: **1 байт** на sample (нулевой counter без TS); worst: -17 байт (double + ts) — почти не хуже фиксированных 16. - -Ветвление при записи: `isnan` → `==0.0` → `trunc(val)==val && val>0` -(uint-фастпас) → `float32-fits-double` → `double`. - -**Числа.** Alloc 5.02 → 3.77 MiB (×3.65). Скорость parse чуть просела: -45.8 → 47.8 ms. Видимо именно из-за добавленных ветвлений в hot loop. - -### 4.3. `4656f7137` «initial encoding» — оба POC + первая нормализация - -Свели обе схемы вместе и привели код к виду, в котором уже можно -писать read-декодер. Read-декодера в нём ещё нет (`MarkedMetric::occupied_size()` -оставлен с FIXME-комментом и `bytes[]` placeholder), а `read()` ничего -разумного не делает. Поэтому числа Read/Pass для него у тебя «N/A» -в CSV (только parse и alloc). - -Замеры на cehp: parse 47.5 ms, alloc 4.15 MiB (×3.31). Это и есть -«дешёвая фаза» — мы максимально экономим память, не заботясь о -скорости чтения. - -### 4.4. `0027219f0` «read + passing all tests» - -**Главное архитектурное решение.** `MetricMarkupBuffer` перестал быть -одним сплошным `Vector`. Стало два контейнера: - -```cpp -BareBones::Vector metric_buffer_; // фиксированный header per metric -BareBones::Vector bytes_buffer_; // variable tail (varint count + labels + sample) -``` - -`MarkedMetric { uint64 hash; uint32 base_offset; uint32 data_offset; }` — фиксированные **16 байт**. - -**Зачем это.** -- `++Iterator` теперь это `++ptr_` по массиву `MarkedMetric*`. До этого нужно было читать `occupied_size()`, который для упакованного формата требовал бы декодировать tail только чтобы посчитать длину. -- На `metric_buffer_` можно сделать `std::sort` без перетаскивания variable tail. -- Hash, base/data offsets лежат плотно — кеш-friendly при чисто «по hash» проходах. - -**Кроме этого.** -- Реализован полный `Metric::read(ts)`: декодирует varint count → распаковывает labels по layout-байту → разбирает sample по маркеру. Под `__name__` подставляет `Prometheus::kMetricLabelName`. -- `default_timestamp` поднят на уровень `Scraper` (один на дамп) и подставляется в `read()`, если у sample не было `has_ts`. Это минус 8 байт на «обычную» метрику. - -**Числа.** Read-декодер «как написан» — медленнее baseline (16.9 ms vs 10.9 на cehp), потому что цикл декодирования label'а делает `memcpy(&v, ptr, sz+1)` с **переменной длиной**, и компилятор не может развернуть это в простую загрузку. Pass тоже просел (38.7 vs 27.0). Alloc — практически тот же 4.13 MiB. Это «честный» промежуточный шаг: формат заработал end-to-end, но дорого по времени. Тесты идут зелёные. - -### 4.5. `9b62e6ae` «read/write optimization» — самый важный коммит по скорости - -**Read.** -- `read_val_partial(ptr, sz)` развёрнут на 4 ветки: sz=0 → один байт, sz=1 → `b0|b1<<8`, sz=2 → `b0|b1<<8|b2<<16`, sz=3 → `memcpy 4 байт`. Это убирает `memcpy` с переменным размером. -- `decode_varint` и `read_val_partial` — `PROMPP_ALWAYS_INLINE`. -- Switch по типу sample отсортирован по частоте (`uint8/zero/uint16/uint32/staleNaN/float/double`). -- Для uint16 — ручная сборка из двух байт со сдвигом (а не memcpy). - -**Write.** Та же идея, что в read, только наоборот: -- `add_count`: вместо `push_back(tmp_array, len)` стало `bytes_buffer_.resize(size + N); *out = …` — пишем сразу в буфер, без промежуточного массива. Размер `N` известен в каждой ветке. -- `add_label`: `resize(size + 17)` (worst case заранее), пишем 4 поля по `memcpy(out, &val, sz+1)`, в конце `resize(offset + bytes_needed)` — «over-allocate then shrink». Это позволяет компилятору эмитить безусловные `mov` без проверок границ внутри. -- `encode_size(v)` — заменён на `(31 - countl_zero(v)) >> 3` (LZCNT/BSR), вместо каскада `if`-ов. - -**Числа.** Cehp: read **16.9 → 7.9 ms (×2.14)**, pass 38.7 → 34.3, parse 50.0 → 46.1. M3: read **4.98 → 1.29 ms (×3.86)**. - -### 4.6. `aca48508` «sample optimization» - -Применили тот же приём `resize+write+shrink` к `add_sample`: - -```cpp -constexpr uint32_t max_sample_bytes = 1 + sizeof(Sample); // 17 -bytes_buffer_.resize(offset + max_sample_bytes); -char* out = ...; *out++ = marker; std::memcpy(out, &val, sizeof(T)); -if (has_ts) { memcpy(out, &ts, 8); out += 8; } -bytes_buffer_.resize(offset + (out - start)); -``` - -Плюс: timestamp кодируется **после** value (раньше — до). На декодер -влияет минимально, но позволяет использовать один общий `out` курсор -без условных переустановок. - -**Числа.** Cehp: pass 34.3 → 33.6, read небольшой шум. M3: pass 16.9 → 16.3. - -### 4.7. `26d0af47` «scraper read benchmark 2 shards» - -Только бенчмарк: `if (metric.hash() % 2 == 0) metric.read(ts);`. Это -меняет **базовую цифру** read и pass — здесь у тебя в CSV вторая -«серия» с новым 3f0b2bd72-baseline (33 ms на pass cehp, 11.3 ms m3). -Все три числа после этого замерены в этой новой системе координат. - -### 4.8. `2036d7ea` «MetricParser → parse_metric method» - -Ранее на каждую метрику конструировался -`MetricParser{parser_, markup_buffer_, labels_, global_offset, default_timestamp}.parse()`. -У вложенного класса с ссылками на 4 объекта компилятор не всегда может -полноценно заинлайнить `parse()`. - -После рефакторинга `parse_metric()` — обычный метод `Scraper`, помеченный -`PROMPP_ALWAYS_INLINE`. Все ссылки — это просто члены того же класса. - -**Числа.** Pass cehp 34.0 → 33.2, m3 17.4 → 16.3. Виден реальный, хоть и небольшой, прирост. - -### 4.9. `1113d04b` «parse_metric refactoring» - -Чистая косметика: вынес `process_labels_buffer`, `sort_and_filter_labels`, -`append_labels_hash` из тела `parse_metric` в отдельные методы. Поведение -и численно — то же. - -## 5. Что зашло, а что нет - -| Коммит | Идея | Память | Скорость parse | Скорость read | Стоит включать в статью | -|---|---|---|---|---|---| -| 4656f7137 (POC labels varint) | переменная упаковка 4 полей label'а + offset relative + дроп `__name__` | ✅ ×2.74 | ⛔ просел | n/a (нет read) | **Да, ключевой шаг** | -| 4656f7137 (POC sample) | 1-байт маркер + 0..8 байт value + опц. ts | ✅ ×3.65 | ⛔ ещё просел | n/a | **Да** | -| 4656f7137 (initial encoding) | свод POC + relative-offset через `base_offset` + дроп overshoot reserve | ✅ ×3.31 (стабильное) | ⛔ медленный pass | ⛔ read «битый» | **Да, как «end of POC»** | -| 0027219f0 | split на `metric_buffer_`+`bytes_buffer_` + рабочий read | = | ⛔ дальше просел | ⛔ медленный | **Да, объясняет архитектуру и трейдоф** | -| 9b62e6ae | read_val_partial unroll, encode_size через LZCNT, resize+write+shrink на write | = | ✅ -10..15% | ✅ ×2.1 (cehp), ×3.9 (m3) | **Да, главный «win-back» по скорости** | -| aca48508 | тот же приём для samples | = | ≈ | ≈ | средне, упомянуть как «продолжение приёма» | -| 26d0af47 | bench 2 shards | n/a | n/a | n/a | методологический момент в статье | -| 2036d7ea | inline-friendly parse_metric | = | ✅ -2..5% | ≈ | **Да, как «чистый» рефактор с замером** | -| 1113d04b | косметика | = | ≈ | ≈ | можно опустить | - -**Итог по серии 1 vs baseline (cehp):** alloc ×3.33, read ×1.33, plain pass ×0.81 — то есть по памяти выиграли ~3.3×, по времени read получили буст ~1.3×, но потеряли ~20% на pass. - -**Итог по серии 1 vs baseline (m3):** alloc ×3.33, read ×1.42, plain pass ×0.72 — m3 «короткий и плотный», поэтому фиксированные накладные новых форматов видны заметнее. - -## 6. Стенд для повторных замеров и трейсов - -### 6.1. Что нужно зафиксировать - -1. Один и тот же бенчмарк-cpp на всех коммитах. Текущий `pp/wal/benchmarks/scraper_benchmark.cpp` уже идеально подходит — он есть в исходном виде в каждом из 8 коммитов после `3f0b2bd72`. Чтобы не зависеть от его эволюции, копируем последнюю версию (с 2-shards read) во временное место и применяем перед сборкой каждой ревизии. -2. Один и тот же набор фикстур. Сейчас в репо лежит только `pp/kube-api.metrics`. Для статьи минимум 2 файла: - - «cehp-runner-like» — длинные labels, разнообразные значения (можно использовать текущий `kube-api.metrics`). - - «m3-like» — плотный, в основном integer counters (можно сделать дамп с `node_exporter` или `cadvisor`, либо синтезировать). - - Опционально третий — пограничный (много float-ов с timestamp'ами). -3. `-c opt --copt=-march=native`, без ASan, без TSan. `pp-workflow.mdc` явно требует `-c opt` для перфметрик. -4. Tracy уже завязан в код (`profiling/profiling.h`, `ZoneScoped` в обеих бенч-функциях). Достаточно собрать с `--//bazel/toolchain:profiling=1` и подключиться `tracy-profiler`-ом. - -### 6.2. Скрипт прогона по ревизиям (черновик) - -```bash -# bench_sweep.sh — запускать из /prompp/pp -set -euo pipefail -COMMITS=( - 3f0b2bd72 # baseline - 4656f7137 # initial encoding (POC merged) - 0027219f0 # read + passing - 9b62e6ae # read/write opt - aca48508 # sample opt - 26d0af47 # 2 shards bench - 2036d7ea # parse_metric method - 1113d04b # parse_metric refactor -) - -FIXTURES=( - "cehp:performance_tests/test_data/cehp.metrics" - "m3:performance_tests/test_data/m3.metrics" -) - -mkdir -p bench_out -cp wal/benchmarks/scraper_benchmark.cpp /tmp/scraper_benchmark.cpp.frozen - -for h in "${COMMITS[@]}"; do - git checkout -q "$h" - cp /tmp/scraper_benchmark.cpp.frozen wal/benchmarks/scraper_benchmark.cpp - - bazel build -c opt --copt=-march=native //wal/benchmarks:scraper - - for kv in "${FIXTURES[@]}"; do - name="${kv%%:*}"; file="${kv##*:}" - out="bench_out/${h}_${name}.json" - ./bazel-bin/wal/benchmarks/scraper \ - --benchmark_repetitions=10 \ - --benchmark_min_time=1s \ - --benchmark_time_unit=ns \ - --benchmark_format=json \ - --benchmark_out="$out" \ - --benchmark_context=prom_scraper_file="$file" \ - --benchmark_filter='Parser|ScraperParse|ScraperRead' - done - - git checkout -q -- wal/benchmarks/scraper_benchmark.cpp -done - -git checkout -q - -``` - -Парсилка JSON-ов в одну CSV — отдельный шаг (`benchmark_repetitions=10` + `aggregate_name == "min"`). - -### 6.3. Tracy-прогоны для статьи - -| Ревизия | Что показать | Идея для скриншота | -|---|---|---| -| `3f0b2bd72` | baseline | один большой `ZoneScoped` в `ScraperParse`, плоско | -| `4656f7137` | POC: память упала, скорость просела | те же зоны, но видно широкие зелёные интервалы внутри `parse_metric` (ветвление в `add_label` / `add_sample`) | -| `0027219f0` | read заработал, но медленный | `ScraperRead` зона: показать дорогой `memcpy` с переменной длиной | -| `9b62e6ae` | главный win-back | те же `read_label` / `add_label` зоны, но в 2–3× уже | -| `2036d7ea` | inlining | пропал отдельный кадр конструктора `MetricParser` | - -Сборка под Tracy: -```bash -bazel build -c opt --copt=-march=native --//bazel/toolchain:profiling=1 //wal/benchmarks:scraper -``` - -В коде уже стоят `ZoneScoped` в `Parser`/`ScraperParse`/`ScraperRead`, и -есть закомментированные `ZoneScopedN("Scraper::parse")` / -`ZoneScopedN("MetricParser::parse")` / `ZoneScopedN("Metric::read()")` в -`scraper.h` (видно в `9b62e6ae` / `aca48508` диффах). Для трейсов их -нужно включить локально — это даст разбивку по фазам. - -Полезно ещё добавить более тонкие зоны (на время статьи, не в master): - -- внутри `parse_metric`: `ZoneScopedN("collect_labels")`, `ZoneScopedN("sort+hash")`, `ZoneScopedN("encode_count")`, `ZoneScopedN("encode_labels")`, `ZoneScopedN("encode_sample")`; -- внутри `Metric::read`: `ZoneScopedN("decode_count")`, `ZoneScopedN("decode_labels")`, `ZoneScopedN("decode_sample")`. - -С такими зонами трейс на 9b62e6ae против 0027219f0 наглядно покажет, -что выигрыш сидит именно в `decode_labels` и `encode_labels`/`encode_sample`. - -### 6.4. Валидация корректности на каждом шаге - -`pp-testing.mdc` требует ASan-прогон тестов перед мерджем; для статьи -это тоже подстраховка от скрытых багов в записи/чтении формата: - -```bash -bazel test -c dbg --asan //wal/hashdex/scraper:scraper_test -``` - -Для `4656f7137` тесты падают (по описанию коммита, тесты «прошли все» -только в `0027219f0`). Это можно использовать честно — «после POC -тесты красные, мы сначала делаем рабочую сериализацию, потом -разгоняем». - -### 6.5. Что ещё имеет смысл достать перед публикацией - -- **Распределение размеров полей** на тестовых дампах: гистограмма `name.length`, `value.length`, `offset_relative_to_metric` — чтобы доказать читателю гипотезу «всё умещается в 1–2 байта». Это `xxd`/python-скрипт по файлу. -- **Распределение sample-типов**: сколько uint8/uint16/uint32/double/NaN. Это и обоснует выбор маркеров. -- **Ratio**: средний размер метрики до/после в байтах. Можно прямо в бенчмарке вывести `bytes_count() / metric_count`. - -## 7. CSV из исходных замеров (нормализованная) - -Колонки: `Parse` — `BenchmarkScraperParse, ns`; `Pass` — `Plain Scraper Time, ns` (parse - parser); `Read` — `BenchmarkScraperRead, ns`; `Alloc` — `ScraperAllocMem, Mi`. В скобках — отношение к baseline. - -### Серия 1 (baseline `3f0b2bd72`) - -#### cehp-runner - -| Commit | Key change | Parse ns (×) | Pass ns (×) | Read ns (×) | Alloc Mi (×) | -|---|---|---:|---:|---:|---:| -| `3f0b2bd72` | baseline | 38171007 (1.00) | 27080951 (1.00) | 10927821 (1.00) | 13.746 (1.00) | -| `labels varint` | labels varint | 45775620 (0.83) | n/a | n/a | 5.01953 (2.74) | -| `sample encoding` | sample encoding | 47803708 (0.80) | n/a | n/a | 3.76953 (3.65) | -| `4656f7137` | initial encoding | 47564602 (0.80) | n/a | n/a | 4.14844 (3.31) | -| `0027219f0` | reading + extra offset added | 50035042 (0.76) | 38723316 (0.70) | 16912478 (0.65) | 4.12891 (3.33) | -| `9b62e6ae` | r/w optimizations | 46106593 (0.83) | 34340283 (0.79) | 7887484 (1.39) | 4.12891 (3.33) | -| `aca48508` | sample opt | 44739040 (0.85) | 33586988 (0.81) | 8198244 (1.33) | 4.12891 (3.33) | - -#### m3 - -| Commit | Key change | Parse ns (×) | Pass ns (×) | Read ns (×) | Alloc Mi (×) | -|---|---|---:|---:|---:|---:| -| `3f0b2bd72` | baseline | 22774389 (1.00) | 11749549 (1.00) | 1791641 (1.00) | 13.746 (1.00) | -| `labels varint` | labels varint | 28791658 (0.79) | n/a | n/a | 5.01953 (2.74) | -| `sample encoding` | sample encoding | 31195012 (0.73) | n/a | n/a | 3.76953 (3.65) | -| `4656f7137` | initial encoding | 28740757 (0.79) | n/a | n/a | 4.14844 (3.31) | -| `0027219f0` | reading + 2-shards bench update | 26255763 (0.87) | 14931157 (0.79) | 4980513 (0.36) | 4.12891 (3.33) | -| `9b62e6ae` | r/w optimizations | 28865275 (0.79) | 16914917 (0.69) | 1287539 (1.39) | 4.12891 (3.33) | -| `aca48508` | sample opt | 28900335 (0.79) | 16259238 (0.72) | 1265548 (1.42) | 4.12891 (3.33) | - -### Серия 2 (новый baseline `3f0b2bd72` после `26d0af47`) - -#### cehp-runner - -| Commit | Key change | Parse ns (×) | Pass ns (×) | Read ns (×) | Alloc Mi (×) | -|---|---|---:|---:|---:|---:| -| `3f0b2bd72` | baseline | 43942332 (1.00) | 32910757 (1.00) | 6080402 (1.00) | 13.746 (1.00) | -| `26d0af47` | 2 shards bench / state after sample opt | 45321131 (0.97) | 34085663 (0.97) | 3535429 (1.72) | 4.129 (3.33) | -| `2036d7ea` | MetricParser → parse_metric method | 44676447 (0.98) | 33235111 (0.99) | 3896916 (1.56) | 4.129 (3.33) | -| `1113d04b` | parse_metric refactoring | 45107031 (0.97) | 33783974 (0.97) | 3791985 (1.60) | 4.129 (3.33) | - -#### m3 - -| Commit | Key change | Parse ns (×) | Pass ns (×) | Read ns (×) | Alloc Mi (×) | -|---|---|---:|---:|---:|---:| -| `3f0b2bd72` | baseline | 22985376 (1.00) | 11277242 (1.00) | 1245206 (1.00) | 13.746 (1.00) | -| `26d0af47` | 2 shards bench / state after sample opt | 29213092 (0.79) | 17359001 (0.65) | 796406 (1.56) | 4.129 (3.33) | -| `2036d7ea` | MetricParser → parse_metric method | 27959700 (0.82) | 16263446 (0.69) | 918037 (1.36) | 4.129 (3.33) | -| `1113d04b` | parse_metric refactoring | 28861682 (0.80) | 17450605 (0.65) | 810836 (1.54) | 4.129 (3.33) | - -## 8. Open questions / TODO перед публикацией - -- [ ] Воспроизвести замеры на свежей машине, на которой будут писаться остальные графики статьи (одна и та же CPU, одна и та же сборка `-c opt --copt=-march=native`). -- [ ] Создать/добавить в репо два публикуемых дампа фикстур (`cehp.metrics`, `m3.metrics`) либо описать, как их получить (например, `curl http://kube-apiserver:443/metrics`). -- [ ] Снять Tracy-трейсы на 5 ревизиях из таблицы 6.3. -- [ ] Сделать гистограммы распределения `name.length` / `value.length` / `value type` на двух фикстурах. -- [ ] Прогнать `bazel test -c dbg --asan //wal/hashdex/scraper:scraper_test` на каждой ревизии, отметить, начиная с какой проходит. diff --git a/pp/wal/benchmarks/scraper_benchmark.cpp b/pp/wal/benchmarks/scraper_benchmark.cpp index 1a3f09177e..81498a7497 100644 --- a/pp/wal/benchmarks/scraper_benchmark.cpp +++ b/pp/wal/benchmarks/scraper_benchmark.cpp @@ -4,8 +4,6 @@ #include "benchmark/statistic.h" #include "primitives/timeseries.h" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Warray-bounds" #include "profiling/profiling.h" #include "wal/hashdex/scraper/scraper.h" diff --git a/pp/wal/hashdex/scraper/encoding.h b/pp/wal/hashdex/scraper/encoding.h index 9dedf4c7ba..4be78d1ac5 100644 --- a/pp/wal/hashdex/scraper/encoding.h +++ b/pp/wal/hashdex/scraper/encoding.h @@ -2,6 +2,9 @@ #include #include +#include +#include +#include #include "bare_bones/bit.h" #include "marked_common.h" @@ -36,16 +39,16 @@ class SampleCodec { using encoding::SampleValueType; const double val = sample.value(); - if (const auto type = layout.value_type(); type == SampleValueType::kUint32) [[likely]] { - out = write_value(out, static_cast(val)); - } else if (type == SampleValueType::kDouble) [[likely]] { - out = write_value(out, val); - } else if (type == SampleValueType::kUint8) [[unlikely]] { + if (const auto type = layout.value_type(); type == SampleValueType::kUint8) [[likely]] { out = write_value(out, static_cast(val)); - } else if (type == SampleValueType::kUint16) [[unlikely]] { + } else if (type == SampleValueType::kUint16) [[likely]] { out = write_value(out, static_cast(val)); + } else if (type == SampleValueType::kUint32) [[likely]] { + out = write_value(out, static_cast(val)); } else if (type == SampleValueType::kFloat) [[unlikely]] { out = write_value(out, static_cast(val)); + } else if (type == SampleValueType::kDouble) [[unlikely]] { + out = write_value(out, val); } if (layout.has_timestamp()) [[unlikely]] { @@ -69,23 +72,23 @@ class SampleCodec { uint64_t chunk; std::memcpy(&chunk, in, sizeof(chunk)); - if (const auto type = layout.value_type(); type == SampleValueType::kUint32) [[likely]] { - val = static_cast(static_cast(chunk)); - in += sizeof(uint32_t); - } else if (type == SampleValueType::kDouble) [[likely]] { - val = std::bit_cast(chunk); - in += sizeof(double); - } else if (type == SampleValueType::kUint8) [[unlikely]] { + if (const auto type = layout.value_type(); type == SampleValueType::kUint8) [[likely]] { val = static_cast(static_cast(chunk)); in += sizeof(uint8_t); - } else if (type == SampleValueType::kUint16) [[unlikely]] { + } else if (type == SampleValueType::kUint16) [[likely]] { val = static_cast(static_cast(chunk)); in += sizeof(uint16_t); + } else if (type == SampleValueType::kUint32) [[likely]] { + val = static_cast(static_cast(chunk)); + in += sizeof(uint32_t); } else if (type == SampleValueType::kFloat) [[unlikely]] { val = static_cast(std::bit_cast(static_cast(chunk))); in += sizeof(float); } else if (type == SampleValueType::kZero) [[unlikely]] { val = 0.0; + } else if (type == SampleValueType::kDouble) [[unlikely]] { + val = std::bit_cast(chunk); + in += sizeof(double); } else { val = Prometheus::kNormalNan; } @@ -158,15 +161,52 @@ class LabelCodec { MarkedLabel label; }; + template + static PROMPP_ALWAYS_INLINE void decode_and_append(const char*& in, const char* base, LabelSet& label_set) noexcept { + uint64_t chunk; + std::memcpy(&chunk, in, sizeof(chunk)); + const auto layout = static_cast(chunk); + + if (layout == 0b01010101) [[likely]] { + label_set.append(std::string_view(base + static_cast(chunk >> 8), static_cast(chunk >> 16)), + std::string_view(base + static_cast(chunk >> 24), static_cast(chunk >> 32))); + in += 5; + return; + } + + if (layout == 0b01000000) [[likely]] { + label_set.append(Prometheus::kMetricLabelName, std::string_view(base, static_cast(chunk >> 8))); + in += 2; + return; + } + + const auto [next, label] = decode_from_chunk(in, chunk, layout); + in = next; + if (label.name.is_reserved_name()) [[unlikely]] { + label_set.append(Prometheus::kMetricLabelName, std::string_view(base + label.value.offset, label.value.length)); + } else { + label_set.append(std::string_view(base + label.name.offset, label.name.length), std::string_view(base + label.value.offset, label.value.length)); + } + } + static DecodeResult decode(const char* in) noexcept { uint64_t chunk; std::memcpy(&chunk, in, sizeof(chunk)); const auto layout = static_cast(chunk); + return decode_from_chunk(in, chunk, layout); + } + + private: + static PROMPP_ALWAYS_INLINE DecodeResult decode_from_chunk(const char* in, const uint64_t chunk, const uint8_t layout) noexcept { if (layout == 0b01010101) [[likely]] { return decode_4_bytes(in, chunk); } + if (layout == 0b01000000) [[likely]] { + return DecodeResult{.next = in + 2, .label = {.name = {.offset = 0, .length = 0}, .value = {.offset = 0, .length = static_cast(chunk >> 8)}}}; + } + if ((layout & 0x0F) == 0) [[likely]] { return decode_value_only(in, chunk, layout); } @@ -174,7 +214,6 @@ class LabelCodec { return decode_generic(++in, layout); } - private: static PROMPP_ALWAYS_INLINE char* encode_value_only(char* out, const uint32_t label_value_offset, const uint32_t label_value_length) noexcept { char* start = out++; @@ -348,16 +387,15 @@ class LayoutCountCodec { }; static PROMPP_ALWAYS_INLINE DecodeResult decode(const char* in) noexcept { - uint64_t chunk; - std::memcpy(&chunk, in, sizeof(chunk)); - LayoutMarker layout{}; - std::memcpy(&layout, &chunk, sizeof(layout)); + std::memcpy(&layout, in, sizeof(layout)); - chunk >>= 8; - const uint64_t mask = (1ULL << BareBones::Bit::to_bits(layout.size_length_in_bytes())) - 1; + if (layout.size_length_in_bytes() == sizeof(uint8_t)) [[likely]] { + return {in + sizeof(layout) + sizeof(uint8_t), layout, static_cast(static_cast(in[1]))}; + } - auto labels_count = static_cast(chunk & mask); + uint32_t labels_count = 0; + std::memcpy(&labels_count, in + sizeof(layout), layout.size_length_in_bytes()); return {in + sizeof(layout) + layout.size_length_in_bytes(), layout, labels_count}; } diff --git a/pp/wal/hashdex/scraper/marked.h b/pp/wal/hashdex/scraper/marked.h index 7409b2a5e9..9f8b0c219e 100644 --- a/pp/wal/hashdex/scraper/marked.h +++ b/pp/wal/hashdex/scraper/marked.h @@ -35,19 +35,10 @@ class Metric { const auto [next_ptr, layout, labels_count] = encoding::LayoutCountCodec::decode(ptr); ptr = next_ptr; - ts.label_set().resize(labels_count); - - auto label_iter = ts.label_set().begin(); + ts.label_set().reserve(labels_count); + const auto buf_ptr = buffer_.data() + item_->base_offset; for (uint32_t i = 0; i < labels_count; ++i) { - const auto [next_ptr, label] = encoding::LabelCodec::decode(ptr); - ptr = next_ptr; - - if (const auto buf_ptr = buffer_.data() + item_->base_offset; label.name.is_reserved_name()) [[unlikely]] { - std::construct_at(label_iter++, Prometheus::kMetricLabelName, std::string_view(buf_ptr + label.value.offset, label.value.length)); - } else { - std::construct_at(label_iter++, std::string_view(buf_ptr + label.name.offset, label.name.length), - std::string_view(buf_ptr + label.value.offset, label.value.length)); - } + encoding::LabelCodec::decode_and_append(ptr, buf_ptr, ts.label_set()); } auto [p, sample] = encoding::SampleCodec::decode(ptr, layout, default_timestamp_);