From 205d1c4c298bbe45f4c34f2c6cfca61d0d4342e0 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Fri, 26 Jun 2026 13:04:05 +0000 Subject: [PATCH 1/5] entrypoint tests --- pp/BUILD | 9 + pp/entrypoint_types/loader_tests.cpp | 82 +++++++ pp/entrypoint_types/lss_tests.cpp | 221 ++++++++++++++++++ pp/entrypoint_types/querier_tests.cpp | 188 +++++++++++++++ pp/entrypoint_types/serialized_data_tests.cpp | 88 +++++++ 5 files changed, 588 insertions(+) create mode 100644 pp/entrypoint_types/loader_tests.cpp create mode 100644 pp/entrypoint_types/lss_tests.cpp create mode 100644 pp/entrypoint_types/querier_tests.cpp create mode 100644 pp/entrypoint_types/serialized_data_tests.cpp diff --git a/pp/BUILD b/pp/BUILD index d67582645..1ae8e34da 100644 --- a/pp/BUILD +++ b/pp/BUILD @@ -198,6 +198,15 @@ cc_library( ], ) +cc_test( + name = "entrypoint_types_test", + srcs = glob(["entrypoint_types/**/*_tests.cpp"]), + deps = [ + ":entrypoint_types", + "@gtest//:gtest_main", + ], +) + cc_library( name = "entrypoint", srcs = glob( diff --git a/pp/entrypoint_types/loader_tests.cpp b/pp/entrypoint_types/loader_tests.cpp new file mode 100644 index 000000000..844c79123 --- /dev/null +++ b/pp/entrypoint_types/loader_tests.cpp @@ -0,0 +1,82 @@ +#include + +#include "bare_bones/streams.h" +#include "entrypoint_types/loader.h" +#include "primitives/label_set.h" +#include "series_data/decoder.h" +#include "series_data/encoder.h" +#include "series_data/encoder/sample.h" +#include "series_data/unloading/unloader.h" + +namespace { + +using entrypoint_types::QueryableEncodingBimap; +using PromPP::Primitives::LabelViewSet; +using series_data::DataStorage; +using series_data::Decoder; +using series_data::Encoder; +using series_data::chunk::DataChunk; +using series_data::encoder::SampleList; +using series_data::unloading::Unloader; + +class RevertableLoaderFixture : public testing::Test { + protected: + DataStorage storage_; + Encoder<> encoder_{storage_}; + Unloader unloader_{storage_}; + BareBones::ShrinkedToFitOStringStream stream_; + QueryableEncodingBimap lss_; + + void SetUp() override { + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 2.0); + encoder_.encode(0, 3, 3.0); + encoder_.encode(0, 4, 4.0); + encoder_.encode(0, 5, 5.0); + + lss_.find_or_emplace(LabelViewSet{{"job", "a"}}); + lss_.build_deferred_indexes(); + } + + [[nodiscard]] auto open_chunk_stream() const { + return storage_.get_asc_integer_stream(storage_.open_chunks[0].encoder.external_index); + } + + [[nodiscard]] SampleList decode_open_chunk() const { return Decoder::decode_chunk(storage_, storage_.open_chunks[0]); } +}; + +TEST_F(RevertableLoaderFixture, LoadFinalizeRestoresUnloadedOpenChunk) { + // Arrange + unloader_.create_snapshot(stream_); + unloader_.unload(); + + // Act + entrypoint_types::RevertableLoader loader{storage_, lss_.ls_id_set().begin(), lss_.ls_id_set().end(), 1}; + loader.load_next(stream_.span()); + loader.load_finalize(); + + // Assert + EXPECT_EQ((SampleList{{1, 1.0}, {2, 2.0}, {3, 3.0}, {4, 4.0}, {5, 5.0}}), decode_open_chunk()); + EXPECT_FALSE(storage_.unloaded_series_bitmap.is_set(0)); +} + +TEST_F(RevertableLoaderFixture, RevertRestoresUnloadedOpenChunk) { + // Arrange + unloader_.create_snapshot(stream_); + unloader_.unload(); + const auto trimmed_stream = open_chunk_stream(); + + entrypoint_types::RevertableLoader loader{storage_, lss_.ls_id_set().begin(), lss_.ls_id_set().end(), 1}; + loader.load_next(stream_.span()); + loader.load_finalize(); + + // Act + loader.revert(); + const auto restored_stream = open_chunk_stream(); + + // Assert + EXPECT_EQ(trimmed_stream, restored_stream); + EXPECT_TRUE(storage_.unloaded_series_bitmap.is_set(0)); +} + +} // namespace diff --git a/pp/entrypoint_types/lss_tests.cpp b/pp/entrypoint_types/lss_tests.cpp new file mode 100644 index 000000000..8ae4c0af0 --- /dev/null +++ b/pp/entrypoint_types/lss_tests.cpp @@ -0,0 +1,221 @@ +#include + +#include +#include +#include + +#include "bare_bones/exception.h" +#include "bare_bones/vector.h" +#include "entrypoint_types/lss.h" +#include "primitives/label_set.h" +#include "series_index/queryable_encoding_bimap.h" + +namespace { + +using entrypoint_types::create_lss; +using entrypoint_types::create_snapshot_lss; +using entrypoint_types::EncodingBimap; +using entrypoint_types::LssType; +using entrypoint_types::QueryableEncodingBimap; +using entrypoint_types::ReallocationsDetector; +using entrypoint_types::ShrinkAwareSnapshotLSS; +using entrypoint_types::SnapshotLSS; +using PromPP::Primitives::LabelViewSet; + +template +using QueryableEncodingBimapCopier = series_index::QueryableEncodingBimapCopier; + +class SnapshotFixture : public testing::Test { + protected: + static constexpr uint32_t kShrinkBoundary = 3U; + + const LabelViewSet ls0_{{"job", "a"}}; + const LabelViewSet ls1_{{"job", "b"}}; + const LabelViewSet ls2_{{"job", "c"}}; + const LabelViewSet ls3_{{"job", "d"}}; + const LabelViewSet ls4_{{"job", "e"}}; + + entrypoint_types::LssVariantPtr create_queryable_lss() const { + auto lss = create_lss(LssType::kQueryableEncodingBimap); + auto& bimap = std::get(*lss); + + bimap.find_or_emplace(ls0_); + bimap.find_or_emplace(ls1_); + bimap.find_or_emplace(ls2_); + bimap.find_or_emplace(ls3_); + bimap.find_or_emplace(ls4_); + + bimap.build_deferred_indexes(); + + return lss; + } + + entrypoint_types::LssVariantPtr create_fixed_lss() const { + auto lss = create_queryable_lss(); + std::get(*lss).set_pending_shrink_boundary(kShrinkBoundary); + + return lss; + } + + entrypoint_types::LssVariantPtr create_shrunk_lss() const { + QueryableEncodingBimap seeded_lss; + seeded_lss.find_or_emplace(ls0_); + seeded_lss.find_or_emplace(ls1_); + seeded_lss.find_or_emplace(ls2_); + seeded_lss.find_or_emplace(ls3_); + seeded_lss.find_or_emplace(ls4_); + + seeded_lss.build_deferred_indexes(); + + auto lss = create_lss(LssType::kQueryableEncodingBimap); + auto& bimap = std::get(*lss); + BareBones::Vector dst_src_ids_mapping; + QueryableEncodingBimapCopier copier(seeded_lss, seeded_lss.sorting_index(), seeded_lss.added_series(), bimap, dst_src_ids_mapping); + copier.copy_added_series_and_build_indexes(); + + std::ignore = bimap.find_or_emplace(ls1_); + std::ignore = bimap.find_or_emplace(ls3_); + std::ignore = bimap.find_or_emplace(ls4_); + bimap.build_deferred_indexes(); + + dst_src_ids_mapping.clear(); + QueryableEncodingBimap lss_copy; + QueryableEncodingBimapCopier shrink_copier(bimap, bimap.sorting_index(), bimap.added_series(), lss_copy, dst_src_ids_mapping); + shrink_copier.copy_added_series_and_build_indexes(); + bimap.set_pending_shrink_boundary(kShrinkBoundary); + const SnapshotLSS resolve_snapshot(lss_copy); + bimap.finalize_copy_and_shrink(resolve_snapshot, dst_src_ids_mapping); + return lss; + } +}; + +TEST(LssTest, CreateLssEncodingBimapSelectsExpectedAlternative) { + // Arrange + + // Act + const auto lss = create_lss(LssType::kEncodingBimap); + + // Assert + EXPECT_TRUE(std::holds_alternative(*lss)); +} + +TEST(LssTest, CreateLssQueryableEncodingBimapSelectsExpectedAlternative) { + // Arrange + + // Act + const auto lss = create_lss(LssType::kQueryableEncodingBimap); + + // Assert + EXPECT_TRUE(std::holds_alternative(*lss)); +} + +TEST(LssTest, CreateLssRejectsUnknownType) { + // Arrange + const auto unknown_type = static_cast(-1); + + // Act + + // Assert + EXPECT_THROW((void)create_lss(unknown_type), BareBones::Exception); +} + +TEST(LssTest, CreateSnapshotFromEncodingBimapProducesPlainSnapshot) { + // Arrange + auto lss = create_lss(LssType::kEncodingBimap); + std::get(*lss).find_or_emplace(LabelViewSet{{"job", "a"}}); + + // Act + const auto snapshot = create_snapshot_lss(*lss); + + // Assert + EXPECT_TRUE(std::holds_alternative(*snapshot)); +} + +TEST_F(SnapshotFixture, SnapshotResolvesNormalQueryableLss) { + // Arrange + auto lss = create_queryable_lss(); + + // Act + const auto snapshot = create_snapshot_lss(*lss); + + // Assert + ASSERT_TRUE(std::holds_alternative(*snapshot)); + EXPECT_EQ(ls0_, std::get(*snapshot)[0]); + EXPECT_EQ(ls4_, std::get(*snapshot)[4]); +} + +TEST_F(SnapshotFixture, SnapshotFromFixedQueryableLssIsShrinkAware) { + // Arrange + auto lss = create_fixed_lss(); + + // Act + const auto snapshot = create_snapshot_lss(*lss); + + // Assert + EXPECT_TRUE(std::holds_alternative(*snapshot)); +} + +TEST_F(SnapshotFixture, ShrinkAwareSnapshotResolvesSurvivingPreBoundarySeries) { + // Arrange + auto lss = create_shrunk_lss(); + + // Act + const auto snapshot = create_snapshot_lss(*lss); + + // Assert + ASSERT_TRUE(std::holds_alternative(*snapshot)); + EXPECT_EQ(ls1_, std::get(*snapshot)[1]); +} + +TEST_F(SnapshotFixture, ShrinkAwareSnapshotHidesDroppedPreBoundarySeries) { + // Arrange + auto lss = create_shrunk_lss(); + + // Act + const auto snapshot = create_snapshot_lss(*lss); + + // Assert + ASSERT_TRUE(std::holds_alternative(*snapshot)); + EXPECT_EQ(0U, std::get(*snapshot)[0].size()); + EXPECT_EQ(0U, std::get(*snapshot)[2].size()); +} + +TEST_F(SnapshotFixture, ShrinkAwareSnapshotResolvesPostBoundarySeries) { + // Arrange + auto lss = create_shrunk_lss(); + + // Act + const auto snapshot = create_snapshot_lss(*lss); + + // Assert + ASSERT_TRUE(std::holds_alternative(*snapshot)); + EXPECT_EQ(ls3_, std::get(*snapshot)[3]); + EXPECT_EQ(ls4_, std::get(*snapshot)[4]); +} + +TEST(LssTest, ReallocationsDetectorReportsReallocOnEmplace) { + // Arrange + QueryableEncodingBimap lss; + ReallocationsDetector detector(lss); + + // Act + lss.find_or_emplace(LabelViewSet{{"job", "a"}}); + + // Assert + EXPECT_TRUE(detector.has_reallocations()); +} + +TEST(LssTest, ReallocationsDetectorStaysQuietWithoutChanges) { + // Arrange + QueryableEncodingBimap lss; + lss.find_or_emplace(LabelViewSet{{"job", "a"}}); + lss.build_deferred_indexes(); + + // Act + ReallocationsDetector detector(lss); + + // Assert + EXPECT_FALSE(detector.has_reallocations()); +} + +} // namespace diff --git a/pp/entrypoint_types/querier_tests.cpp b/pp/entrypoint_types/querier_tests.cpp new file mode 100644 index 000000000..1da40fc46 --- /dev/null +++ b/pp/entrypoint_types/querier_tests.cpp @@ -0,0 +1,188 @@ +#include + +#include +#include +#include +#include +#include + +#include "bare_bones/streams.h" +#include "entrypoint_types/querier.h" +#include "series_data/decoder.h" +#include "series_data/encoder.h" +#include "series_data/unloading/loader.h" +#include "series_data/unloading/unloader.h" + +namespace { + +using BareBones::Encoding::Gorilla::STALE_NAN; +using PromPP::Primitives::LabelSetID; +using PromPP::Primitives::Go::Slice; +using series_data::DataStorage; +using series_data::Decoder; +using series_data::Encoder; +using series_data::decoder::DecodeIteratorSentinel; +using series_data::encoder::Sample; +using series_data::encoder::SampleList; +using series_data::unloading::Loader; +using series_data::unloading::Unloader; +using InstantQuerierWrapper = entrypoint_types::InstantQuerierWithArgumentsWrapper, std::span>; +using RangeQuery = series_data::querier::Query>; + +class SerializedDataPtrStorage { + public: + ~SerializedDataPtrStorage() { + if (constructed_) { + std::destroy_at(ptr()); + } + } + + [[nodiscard]] entrypoint_types::SerializedDataPtr* ptr() noexcept { return reinterpret_cast(storage_); } + [[nodiscard]] const entrypoint_types::SerializedDataPtr* ptr() const noexcept { + return reinterpret_cast(storage_); + } + [[nodiscard]] const entrypoint_types::SerializedDataGo* get() const noexcept { return ptr()->get(); } + + void mark_constructed() noexcept { constructed_ = true; } + + private: + alignas(entrypoint_types::SerializedDataPtr) std::byte storage_[sizeof(entrypoint_types::SerializedDataPtr)]; + bool constructed_{false}; +}; + +class InstantQuerierWrapperFixture : public testing::Test { + protected: + DataStorage storage_; + Encoder<> encoder_{storage_}; + BareBones::ShrinkedToFitOStringStream unloaded_chunks_; + std::vector label_set_ids_{0}; + std::vector samples_{Sample{.timestamp = -1, .value = STALE_NAN}}; + + void encode_open_chunk() { + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 2.0); + encoder_.encode(0, 3, 3.0); + encoder_.encode(0, 4, 4.0); + encoder_.encode(0, 5, 5.0); + } + + void unload_open_chunks() { + Unloader unloader{storage_}; + unloader.create_snapshot(unloaded_chunks_); + unloader.unload(); + } + + void load_unloaded_chunks() { + Loader loader{storage_, label_set_ids_, static_cast(label_set_ids_.size())}; + loader.load_next(unloaded_chunks_.span()); + loader.load_finalize(); + } +}; + +TEST_F(InstantQuerierWrapperFixture, QueryReturnsSampleBeforeTimestamp) { + // Arrange + encode_open_chunk(); + std::span samples_view{samples_}; + InstantQuerierWrapper wrapper{storage_, label_set_ids_, 3, samples_view}; + + // Act + wrapper.query(); + + // Assert + EXPECT_EQ((Sample{.timestamp = 3, .value = 3.0}), samples_[0]); + EXPECT_FALSE(wrapper.need_loading()); +} + +TEST_F(InstantQuerierWrapperFixture, QueryKeepsDefaultSampleWhenSeriesHasNoPointBeforeTimestamp) { + // Arrange + encoder_.encode(0, 10, 10.0); + std::span samples_view{samples_}; + InstantQuerierWrapper wrapper{storage_, label_set_ids_, 5, samples_view}; + + // Act + wrapper.query(); + + // Assert + EXPECT_EQ((Sample{.timestamp = -1, .value = STALE_NAN}), samples_[0]); + EXPECT_FALSE(wrapper.need_loading()); +} + +TEST_F(InstantQuerierWrapperFixture, QueryRequestsLoadingForUnloadedSeriesThenFinalizeReturnsSample) { + // Arrange + encode_open_chunk(); + unload_open_chunks(); + + std::span samples_view{samples_}; + InstantQuerierWrapper wrapper{storage_, label_set_ids_, 3, samples_view}; + + // Act + wrapper.query(); + const auto need_loading = wrapper.need_loading(); + const auto series_to_load_0 = wrapper.series_to_load().is_set(0); + load_unloaded_chunks(); + wrapper.query_finalize(); + + // Assert + ASSERT_TRUE(need_loading); + EXPECT_TRUE(series_to_load_0); + EXPECT_EQ((Sample{.timestamp = 3, .value = 3.0}), samples_[0]); +} + +class RangeQuerierWrapperFixture : public testing::Test { + protected: + DataStorage storage_; + Encoder<> encoder_{storage_}; + SerializedDataPtrStorage serialized_data_; + + RangeQuery query_for(LabelSetID label_set_id, int64_t min, int64_t max) { + Slice label_set_ids; + label_set_ids.push_back(label_set_id); + return RangeQuery{.time_interval{.min = min, .max = max}, .label_set_ids = std::move(label_set_ids)}; + } + + [[nodiscard]] SampleList decode_chunk(uint32_t chunk_id) const { + SampleList decoded; + std::ranges::copy((*serialized_data_.ptr())->iterator(chunk_id), DecodeIteratorSentinel{}, std::back_inserter(decoded)); + return decoded; + } +}; + +TEST_F(RangeQuerierWrapperFixture, QuerySerializesMatchingOpenChunk) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 2.0); + encoder_.encode(0, 3, 3.0); + encoder_.encode(0, 4, 4.0); + encoder_.encode(0, 5, 5.0); + auto query = query_for(0, 2, 4); + entrypoint_types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_.ptr()}; + + // Act + wrapper.query(); + serialized_data_.mark_constructed(); + const auto decoded = decode_chunk(0); + + // Assert + ASSERT_FALSE(wrapper.need_loading()); + ASSERT_NE(nullptr, serialized_data_.get()); + ASSERT_EQ(1U, serialized_data_.get()->get_chunks_view().size()); + EXPECT_EQ((SampleList{{1, 1.0}, {2, 2.0}, {3, 3.0}, {4, 4.0}, {5, 5.0}}), decoded); +} + +TEST_F(RangeQuerierWrapperFixture, QuerySerializesEmptyResultWhenSeriesDoesNotMatchInterval) { + // Arrange + encoder_.encode(0, 10, 10.0); + auto query = query_for(0, 1, 5); + entrypoint_types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_.ptr()}; + + // Act + wrapper.query(); + serialized_data_.mark_constructed(); + + // Assert + ASSERT_FALSE(wrapper.need_loading()); + ASSERT_NE(nullptr, serialized_data_.get()); + EXPECT_EQ(0U, serialized_data_.get()->get_chunks_view().size()); +} + +} // namespace diff --git a/pp/entrypoint_types/serialized_data_tests.cpp b/pp/entrypoint_types/serialized_data_tests.cpp new file mode 100644 index 000000000..5dc239851 --- /dev/null +++ b/pp/entrypoint_types/serialized_data_tests.cpp @@ -0,0 +1,88 @@ +#include + +#include + +#include "entrypoint_types/serialized_data.h" +#include "series_data/chunk_finalizer.h" +#include "series_data/decoder/traits.h" +#include "series_data/encoder.h" +#include "series_data/encoder/sample.h" +#include "series_data/querier/querier.h" + +namespace { + +using series_data::ChunkFinalizer; +using series_data::DataStorage; +using series_data::Encoder; +using series_data::decoder::DecodeIteratorSentinel; +using series_data::encoder::Sample; +using series_data::encoder::SampleList; +using series_data::querier::Querier; +using Query = series_data::querier::Query>; + +class SerializedDataFixture : public testing::Test { + protected: + DataStorage storage_; + Encoder<> encoder_{storage_}; + Querier querier_{storage_}; + + [[nodiscard]] static SampleList decode_chunk(const entrypoint_types::SerializedDataGo& data, uint32_t chunk_id) { + SampleList decoded; + std::ranges::copy(data.iterator(chunk_id), DecodeIteratorSentinel{}, std::back_inserter(decoded)); + return decoded; + } +}; + +TEST_F(SerializedDataFixture, EmptyQueriedChunkListProducesNoChunks) { + // Arrange + + // Act + entrypoint_types::SerializedDataGo data{storage_, {}}; + + // Assert + EXPECT_EQ(0U, data.get_chunks_view().size()); +} + +TEST_F(SerializedDataFixture, RoundTripsQueriedOpenChunk) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 2.0); + encoder_.encode(0, 3, 3.0); + encoder_.encode(0, 4, 4.0); + encoder_.encode(0, 5, 5.0); + const auto& queried_chunks = querier_.query(Query{.time_interval{.min = 1, .max = 5}, .label_set_ids = {0}}); + + // Act + entrypoint_types::SerializedDataGo data{storage_, queried_chunks}; + const auto next_series = data.next(); + const auto decoded = decode_chunk(data, 0); + + // Assert + ASSERT_EQ(1U, data.get_chunks_view().size()); + EXPECT_EQ(0U, next_series.first); + EXPECT_EQ((SampleList{{1, 1.0}, {2, 2.0}, {3, 3.0}, {4, 4.0}, {5, 5.0}}), decoded); +} + +TEST_F(SerializedDataFixture, RoundTripsQueriedFinalizedChunk) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 2.0); + encoder_.encode(0, 3, 3.0); + encoder_.encode(0, 4, 4.0); + encoder_.encode(0, 5, 5.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 10, 10.0); + const auto& queried_chunks = querier_.query(Query{.time_interval{.min = 1, .max = 5}, .label_set_ids = {0}}); + + // Act + entrypoint_types::SerializedDataGo data{storage_, queried_chunks}; + const auto next_series = data.next(); + const auto decoded = decode_chunk(data, 0); + + // Assert + ASSERT_EQ(1U, data.get_chunks_view().size()); + EXPECT_EQ(0U, next_series.first); + EXPECT_EQ((SampleList{{1, 1.0}, {2, 2.0}, {3, 3.0}, {4, 4.0}, {5, 5.0}}), decoded); +} + +} // namespace From c0f0c48baacbcf32c51e5443ab0cf2698c16035c Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Mon, 29 Jun 2026 13:25:31 +0000 Subject: [PATCH 2/5] refine tests --- pp/entrypoint/go_constants.cpp | 24 --- pp/entrypoint/series_data_data_storage.cpp | 1 - pp/entrypoint_types/go_constants_tests.cpp | 33 ++++ pp/entrypoint_types/loader_tests.cpp | 94 +++++++++- pp/entrypoint_types/lss_tests.cpp | 100 +++++------ pp/entrypoint_types/querier_tests.cpp | 170 +++++++++++++++--- pp/entrypoint_types/serialized_data_tests.cpp | 40 ++++- 7 files changed, 355 insertions(+), 107 deletions(-) delete mode 100644 pp/entrypoint/go_constants.cpp create mode 100644 pp/entrypoint_types/go_constants_tests.cpp diff --git a/pp/entrypoint/go_constants.cpp b/pp/entrypoint/go_constants.cpp deleted file mode 100644 index a9328ab5b..000000000 --- a/pp/entrypoint/go_constants.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#include "entrypoint_types/go_constants.h" -#include "entrypoint_types/serialized_data.h" -#include "metrics/storage.h" -#include "prometheus/relabeler.h" -#include "wal/output_decoder.h" -#include "wal/segment_samples_storage.h" - -namespace { - -static_assert(sizeof(std::vector) == Sizeof_StdVector); -static_assert(sizeof(BareBones::Vector) == Sizeof_BareBonesVector); -static_assert(sizeof(roaring::Roaring) == Sizeof_RoaringBitset); - -static_assert(sizeof(PromPP::Prometheus::Relabel::InnerSeries) == Sizeof_InnerSeries); - -static_assert(sizeof(entrypoint_types::SerializedDataIterator) == Sizeof_SerializedDataIterator); - -static_assert(sizeof(metrics::Storage::Iterator) == Sizeof_MetricsIterator); - -static_assert(sizeof(PromPP::WAL::SegmentSamplesStorage) == Sizeof_SegmentSamplesStorage); -static_assert(sizeof(PromPP::WAL::ProtobufEncoder) == Sizeof_RemoteWriteMessageEncoder); -static_assert(sizeof(PromPP::WAL::SegmentSamplesStorageList::Iterator) == Sizeof_SegmentSamplesStorageListIterator); - -} // namespace \ No newline at end of file diff --git a/pp/entrypoint/series_data_data_storage.cpp b/pp/entrypoint/series_data_data_storage.cpp index 2e6cd088b..19f07a939 100644 --- a/pp/entrypoint/series_data_data_storage.cpp +++ b/pp/entrypoint/series_data_data_storage.cpp @@ -16,7 +16,6 @@ #include "series_data/querier/querier.h" #include "series_data/unloading/loader.h" #include "series_data/unloading/unloader.h" -#include "series_index/querier/selector_querier.h" using entrypoint_types::DataStoragePtr; using entrypoint_types::QueryableEncodingBimap; diff --git a/pp/entrypoint_types/go_constants_tests.cpp b/pp/entrypoint_types/go_constants_tests.cpp new file mode 100644 index 000000000..0ccf874e2 --- /dev/null +++ b/pp/entrypoint_types/go_constants_tests.cpp @@ -0,0 +1,33 @@ +#include + +#include + +#include "bare_bones/vector.h" +#include "entrypoint_types/go_constants.h" +#include "entrypoint_types/serialized_data.h" +#include "metrics/storage.h" +#include "prometheus/relabeler.h" +#include "wal/output_decoder.h" +#include "wal/segment_samples_storage.h" + +namespace { + +TEST(GoConstantsTest, CompileTimeSizesMatchConstants) { + static_assert(sizeof(std::vector) == Sizeof_StdVector); + static_assert(sizeof(BareBones::Vector) == Sizeof_BareBonesVector); + static_assert(sizeof(roaring::Roaring) == Sizeof_RoaringBitset); + + static_assert(sizeof(PromPP::Prometheus::Relabel::InnerSeries) == Sizeof_InnerSeries); + + static_assert(sizeof(entrypoint_types::SerializedDataIterator) == Sizeof_SerializedDataIterator); + + static_assert(sizeof(metrics::Storage::Iterator) == Sizeof_MetricsIterator); + + static_assert(sizeof(PromPP::WAL::SegmentSamplesStorage) == Sizeof_SegmentSamplesStorage); + static_assert(sizeof(PromPP::WAL::ProtobufEncoder) == Sizeof_RemoteWriteMessageEncoder); + static_assert(sizeof(PromPP::WAL::SegmentSamplesStorageList::Iterator) == Sizeof_SegmentSamplesStorageListIterator); + + SUCCEED(); +} + +} // namespace diff --git a/pp/entrypoint_types/loader_tests.cpp b/pp/entrypoint_types/loader_tests.cpp index 844c79123..75161d0de 100644 --- a/pp/entrypoint_types/loader_tests.cpp +++ b/pp/entrypoint_types/loader_tests.cpp @@ -38,11 +38,22 @@ class RevertableLoaderFixture : public testing::Test { lss_.build_deferred_indexes(); } - [[nodiscard]] auto open_chunk_stream() const { - return storage_.get_asc_integer_stream(storage_.open_chunks[0].encoder.external_index); + void encode_more(uint32_t ls_id, const LabelViewSet& label_set, const SampleList& samples) { + for (const auto& sample : samples) { + encoder_.encode(ls_id, sample.timestamp, sample.value); + } + + lss_.find_or_emplace(label_set); + lss_.build_deferred_indexes(); + } + + [[nodiscard]] auto open_chunk_stream(uint32_t ls_id) const { + return storage_.get_asc_integer_stream(storage_.open_chunks[ls_id].encoder.external_index); } - [[nodiscard]] SampleList decode_open_chunk() const { return Decoder::decode_chunk(storage_, storage_.open_chunks[0]); } + [[nodiscard]] SampleList decode_open_chunk(uint32_t ls_id) const { + return Decoder::decode_chunk(storage_, storage_.open_chunks[ls_id]); + } }; TEST_F(RevertableLoaderFixture, LoadFinalizeRestoresUnloadedOpenChunk) { @@ -56,7 +67,7 @@ TEST_F(RevertableLoaderFixture, LoadFinalizeRestoresUnloadedOpenChunk) { loader.load_finalize(); // Assert - EXPECT_EQ((SampleList{{1, 1.0}, {2, 2.0}, {3, 3.0}, {4, 4.0}, {5, 5.0}}), decode_open_chunk()); + EXPECT_EQ((SampleList{{1, 1.0}, {2, 2.0}, {3, 3.0}, {4, 4.0}, {5, 5.0}}), decode_open_chunk(0)); EXPECT_FALSE(storage_.unloaded_series_bitmap.is_set(0)); } @@ -64,7 +75,7 @@ TEST_F(RevertableLoaderFixture, RevertRestoresUnloadedOpenChunk) { // Arrange unloader_.create_snapshot(stream_); unloader_.unload(); - const auto trimmed_stream = open_chunk_stream(); + const auto trimmed_stream = open_chunk_stream(0); entrypoint_types::RevertableLoader loader{storage_, lss_.ls_id_set().begin(), lss_.ls_id_set().end(), 1}; loader.load_next(stream_.span()); @@ -72,11 +83,82 @@ TEST_F(RevertableLoaderFixture, RevertRestoresUnloadedOpenChunk) { // Act loader.revert(); - const auto restored_stream = open_chunk_stream(); + const auto restored_stream = open_chunk_stream(0); // Assert EXPECT_EQ(trimmed_stream, restored_stream); EXPECT_TRUE(storage_.unloaded_series_bitmap.is_set(0)); } +TEST_F(RevertableLoaderFixture, LoadFinalizeLoadsSeriesByBatch) { + // Arrange + encode_more(1, LabelViewSet{{"job", "b"}}, SampleList{{1, 11.0}, {2, 12.0}, {3, 13.0}, {4, 14.0}, {5, 15.0}}); + encode_more(2, LabelViewSet{{"job", "c"}}, SampleList{{1, 21.0}, {2, 22.0}, {3, 23.0}, {4, 24.0}, {5, 25.0}}); + + unloader_.create_snapshot(stream_); + unloader_.unload(); + + entrypoint_types::RevertableLoader loader{storage_, lss_.ls_id_set().begin(), lss_.ls_id_set().end(), 2}; + + // Act + loader.load_next(stream_.span()); + loader.load_finalize(); + + const auto has_second_batch = loader.next_batch(); + loader.load_next(stream_.span()); + loader.load_finalize(); + + const auto has_third_batch = loader.next_batch(); + + // Assert + EXPECT_TRUE(has_second_batch); + EXPECT_FALSE(has_third_batch); + + EXPECT_FALSE(storage_.unloaded_series_bitmap.is_set(0)); + EXPECT_FALSE(storage_.unloaded_series_bitmap.is_set(1)); + EXPECT_FALSE(storage_.unloaded_series_bitmap.is_set(2)); + + EXPECT_EQ((SampleList{{1, 1.0}, {2, 2.0}, {3, 3.0}, {4, 4.0}, {5, 5.0}}), decode_open_chunk(0)); + EXPECT_EQ((SampleList{{1, 11.0}, {2, 12.0}, {3, 13.0}, {4, 14.0}, {5, 15.0}}), decode_open_chunk(1)); + EXPECT_EQ((SampleList{{1, 21.0}, {2, 22.0}, {3, 23.0}, {4, 24.0}, {5, 25.0}}), decode_open_chunk(2)); +} + +TEST_F(RevertableLoaderFixture, RevertRestoresSeriesLoadedAcrossBatches) { + // Arrange + encode_more(1, LabelViewSet{{"job", "b"}}, SampleList{{1, 11.0}, {2, 12.0}, {3, 13.0}, {4, 14.0}, {5, 15.0}}); + encode_more(2, LabelViewSet{{"job", "c"}}, SampleList{{1, 21.0}, {2, 22.0}, {3, 23.0}, {4, 24.0}, {5, 25.0}}); + + unloader_.create_snapshot(stream_); + unloader_.unload(); + const auto trimmed_stream0 = open_chunk_stream(0); + const auto trimmed_stream1 = open_chunk_stream(1); + const auto trimmed_stream2 = open_chunk_stream(2); + + entrypoint_types::RevertableLoader loader{storage_, lss_.ls_id_set().begin(), lss_.ls_id_set().end(), 2}; + + loader.load_next(stream_.span()); + loader.load_finalize(); + + const auto has_second_batch = loader.next_batch(); + loader.load_next(stream_.span()); + loader.load_finalize(); + + // Act + loader.revert(); + const auto restored_stream0 = open_chunk_stream(0); + const auto restored_stream1 = open_chunk_stream(1); + const auto restored_stream2 = open_chunk_stream(2); + + // Assert + ASSERT_TRUE(has_second_batch); + + EXPECT_EQ(trimmed_stream0, restored_stream0); + EXPECT_EQ(trimmed_stream1, restored_stream1); + EXPECT_EQ(trimmed_stream2, restored_stream2); + + EXPECT_TRUE(storage_.unloaded_series_bitmap.is_set(0)); + EXPECT_TRUE(storage_.unloaded_series_bitmap.is_set(1)); + EXPECT_TRUE(storage_.unloaded_series_bitmap.is_set(2)); +} + } // namespace diff --git a/pp/entrypoint_types/lss_tests.cpp b/pp/entrypoint_types/lss_tests.cpp index 8ae4c0af0..2a0cb47b9 100644 --- a/pp/entrypoint_types/lss_tests.cpp +++ b/pp/entrypoint_types/lss_tests.cpp @@ -22,10 +22,52 @@ using entrypoint_types::ShrinkAwareSnapshotLSS; using entrypoint_types::SnapshotLSS; using PromPP::Primitives::LabelViewSet; +TEST(LssTest, CreateLssEncodingBimapSelectsExpectedAlternative) { + // Arrange + + // Act + const auto lss = create_lss(LssType::kEncodingBimap); + + // Assert + EXPECT_TRUE(std::holds_alternative(*lss)); +} + +TEST(LssTest, CreateLssQueryableEncodingBimapSelectsExpectedAlternative) { + // Arrange + + // Act + const auto lss = create_lss(LssType::kQueryableEncodingBimap); + + // Assert + EXPECT_TRUE(std::holds_alternative(*lss)); +} + +TEST(LssTest, CreateLssRejectsUnknownType) { + // Arrange + const auto unknown_type = static_cast(-1); + + // Act + + // Assert + EXPECT_THROW((void)create_lss(unknown_type), BareBones::Exception); +} + +TEST(LssTest, CreateSnapshotFromEncodingBimapProducesPlainSnapshot) { + // Arrange + auto lss = create_lss(LssType::kEncodingBimap); + std::get(*lss).find_or_emplace(LabelViewSet{{"job", "a"}}); + + // Act + const auto snapshot = create_snapshot_lss(*lss); + + // Assert + EXPECT_TRUE(std::holds_alternative(*snapshot)); +} + template using QueryableEncodingBimapCopier = series_index::QueryableEncodingBimapCopier; -class SnapshotFixture : public testing::Test { +class SnapshotLssFixture : public testing::Test { protected: static constexpr uint32_t kShrinkBoundary = 3U; @@ -89,49 +131,7 @@ class SnapshotFixture : public testing::Test { } }; -TEST(LssTest, CreateLssEncodingBimapSelectsExpectedAlternative) { - // Arrange - - // Act - const auto lss = create_lss(LssType::kEncodingBimap); - - // Assert - EXPECT_TRUE(std::holds_alternative(*lss)); -} - -TEST(LssTest, CreateLssQueryableEncodingBimapSelectsExpectedAlternative) { - // Arrange - - // Act - const auto lss = create_lss(LssType::kQueryableEncodingBimap); - - // Assert - EXPECT_TRUE(std::holds_alternative(*lss)); -} - -TEST(LssTest, CreateLssRejectsUnknownType) { - // Arrange - const auto unknown_type = static_cast(-1); - - // Act - - // Assert - EXPECT_THROW((void)create_lss(unknown_type), BareBones::Exception); -} - -TEST(LssTest, CreateSnapshotFromEncodingBimapProducesPlainSnapshot) { - // Arrange - auto lss = create_lss(LssType::kEncodingBimap); - std::get(*lss).find_or_emplace(LabelViewSet{{"job", "a"}}); - - // Act - const auto snapshot = create_snapshot_lss(*lss); - - // Assert - EXPECT_TRUE(std::holds_alternative(*snapshot)); -} - -TEST_F(SnapshotFixture, SnapshotResolvesNormalQueryableLss) { +TEST_F(SnapshotLssFixture, ResolvesNormalQueryableLss) { // Arrange auto lss = create_queryable_lss(); @@ -144,7 +144,7 @@ TEST_F(SnapshotFixture, SnapshotResolvesNormalQueryableLss) { EXPECT_EQ(ls4_, std::get(*snapshot)[4]); } -TEST_F(SnapshotFixture, SnapshotFromFixedQueryableLssIsShrinkAware) { +TEST_F(SnapshotLssFixture, FromFixedQueryableLssIsShrinkAware) { // Arrange auto lss = create_fixed_lss(); @@ -155,7 +155,7 @@ TEST_F(SnapshotFixture, SnapshotFromFixedQueryableLssIsShrinkAware) { EXPECT_TRUE(std::holds_alternative(*snapshot)); } -TEST_F(SnapshotFixture, ShrinkAwareSnapshotResolvesSurvivingPreBoundarySeries) { +TEST_F(SnapshotLssFixture, ShrinkAwareResolvesSurvivingPreBoundarySeries) { // Arrange auto lss = create_shrunk_lss(); @@ -167,7 +167,7 @@ TEST_F(SnapshotFixture, ShrinkAwareSnapshotResolvesSurvivingPreBoundarySeries) { EXPECT_EQ(ls1_, std::get(*snapshot)[1]); } -TEST_F(SnapshotFixture, ShrinkAwareSnapshotHidesDroppedPreBoundarySeries) { +TEST_F(SnapshotLssFixture, ShrinkAwareHidesDroppedPreBoundarySeries) { // Arrange auto lss = create_shrunk_lss(); @@ -180,7 +180,7 @@ TEST_F(SnapshotFixture, ShrinkAwareSnapshotHidesDroppedPreBoundarySeries) { EXPECT_EQ(0U, std::get(*snapshot)[2].size()); } -TEST_F(SnapshotFixture, ShrinkAwareSnapshotResolvesPostBoundarySeries) { +TEST_F(SnapshotLssFixture, ShrinkAwareResolvesPostBoundarySeries) { // Arrange auto lss = create_shrunk_lss(); @@ -193,7 +193,7 @@ TEST_F(SnapshotFixture, ShrinkAwareSnapshotResolvesPostBoundarySeries) { EXPECT_EQ(ls4_, std::get(*snapshot)[4]); } -TEST(LssTest, ReallocationsDetectorReportsReallocOnEmplace) { +TEST(ReallocationsDetectorTest, ReportsReallocOnEmplace) { // Arrange QueryableEncodingBimap lss; ReallocationsDetector detector(lss); @@ -205,7 +205,7 @@ TEST(LssTest, ReallocationsDetectorReportsReallocOnEmplace) { EXPECT_TRUE(detector.has_reallocations()); } -TEST(LssTest, ReallocationsDetectorStaysQuietWithoutChanges) { +TEST(ReallocationsDetectorTest, StaysQuietWithoutChanges) { // Arrange QueryableEncodingBimap lss; lss.find_or_emplace(LabelViewSet{{"job", "a"}}); diff --git a/pp/entrypoint_types/querier_tests.cpp b/pp/entrypoint_types/querier_tests.cpp index 1da40fc46..1edf7fff0 100644 --- a/pp/entrypoint_types/querier_tests.cpp +++ b/pp/entrypoint_types/querier_tests.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -29,27 +30,101 @@ using series_data::unloading::Unloader; using InstantQuerierWrapper = entrypoint_types::InstantQuerierWithArgumentsWrapper, std::span>; using RangeQuery = series_data::querier::Query>; -class SerializedDataPtrStorage { +template +class UninitializedMemory { public: - ~SerializedDataPtrStorage() { - if (constructed_) { + UninitializedMemory() { std::ranges::fill(storage_, kDefaultValue); } + + ~UninitializedMemory() { + if (!has_default_value()) { std::destroy_at(ptr()); } } - [[nodiscard]] entrypoint_types::SerializedDataPtr* ptr() noexcept { return reinterpret_cast(storage_); } - [[nodiscard]] const entrypoint_types::SerializedDataPtr* ptr() const noexcept { - return reinterpret_cast(storage_); + [[nodiscard]] T* ptr() noexcept { return reinterpret_cast(storage_); } + [[nodiscard]] const T* ptr() const noexcept { return reinterpret_cast(storage_); } + [[nodiscard]] T& value() noexcept { return *ptr(); } + [[nodiscard]] const T& value() const noexcept { return *ptr(); } + [[nodiscard]] bool has_default_value() const noexcept { + return std::ranges::all_of(storage_, [](std::byte byte) { return byte == kDefaultValue; }); } - [[nodiscard]] const entrypoint_types::SerializedDataGo* get() const noexcept { return ptr()->get(); } - - void mark_constructed() noexcept { constructed_ = true; } private: - alignas(entrypoint_types::SerializedDataPtr) std::byte storage_[sizeof(entrypoint_types::SerializedDataPtr)]; - bool constructed_{false}; + static constexpr auto kDefaultValue = std::byte{0x5a}; + + alignas(T) std::byte storage_[sizeof(T)]; +}; + +class RangeQuerierUninitializedMemoryFixture : public testing::Test { + protected: + DataStorage storage_; + Encoder<> encoder_{storage_}; + BareBones::ShrinkedToFitOStringStream unloaded_chunks_; + UninitializedMemory serialized_data_memory_; + + RangeQuery query_for(LabelSetID label_set_id, int64_t min, int64_t max) { + Slice label_set_ids; + label_set_ids.push_back(label_set_id); + return RangeQuery{.time_interval{.min = min, .max = max}, .label_set_ids = std::move(label_set_ids)}; + } + + [[nodiscard]] entrypoint_types::SerializedDataPtr* serialized_data_ptr() noexcept { return serialized_data_memory_.ptr(); } + + void unload_open_chunks() { + Unloader unloader{storage_}; + unloader.create_snapshot(unloaded_chunks_); + unloader.unload(); + } + + void load_unloaded_chunks(LabelSetID label_set_id) { + std::vector label_set_ids{label_set_id}; + Loader loader{storage_, label_set_ids, static_cast(label_set_ids.size())}; + loader.load_next(unloaded_chunks_.span()); + loader.load_finalize(); + } }; +TEST_F(RangeQuerierUninitializedMemoryFixture, QueryWritesSerializedDataToPreparedMemory) { + // Arrange + encoder_.encode(0, 1, 1.0); + auto query = query_for(0, 1, 1); + const auto was_default_before_prepare = serialized_data_memory_.has_default_value(); + entrypoint_types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_ptr()}; + + // Act + wrapper.query(); + + // Assert + EXPECT_TRUE(was_default_before_prepare); + EXPECT_FALSE(serialized_data_memory_.has_default_value()); + ASSERT_NE(nullptr, serialized_data_memory_.value().get()); +} + +TEST_F(RangeQuerierUninitializedMemoryFixture, QueryFinalizeWritesSerializedDataToPreparedMemory) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 2.0); + encoder_.encode(0, 3, 3.0); + + unload_open_chunks(); + + auto query = query_for(0, 1, 3); + const auto was_default_before_prepare = serialized_data_memory_.has_default_value(); + entrypoint_types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_ptr()}; + + // Act + wrapper.query(); + const auto need_loading = wrapper.need_loading(); + load_unloaded_chunks(0); + wrapper.query_finalize(); + + // Assert + ASSERT_TRUE(need_loading); + EXPECT_TRUE(was_default_before_prepare); + EXPECT_FALSE(serialized_data_memory_.has_default_value()); + ASSERT_NE(nullptr, serialized_data_memory_.value().get()); +} + class InstantQuerierWrapperFixture : public testing::Test { protected: DataStorage storage_; @@ -79,7 +154,7 @@ class InstantQuerierWrapperFixture : public testing::Test { } }; -TEST_F(InstantQuerierWrapperFixture, QueryReturnsSampleBeforeTimestamp) { +TEST_F(InstantQuerierWrapperFixture, QueryReturnsSampleAtTimestamp) { // Arrange encode_open_chunk(); std::span samples_view{samples_}; @@ -132,7 +207,9 @@ class RangeQuerierWrapperFixture : public testing::Test { protected: DataStorage storage_; Encoder<> encoder_{storage_}; - SerializedDataPtrStorage serialized_data_; + BareBones::ShrinkedToFitOStringStream unloaded_chunks_; + UninitializedMemory serialized_data_memory_; + entrypoint_types::SerializedDataPtr serialized_data_; RangeQuery query_for(LabelSetID label_set_id, int64_t min, int64_t max) { Slice label_set_ids; @@ -142,9 +219,26 @@ class RangeQuerierWrapperFixture : public testing::Test { [[nodiscard]] SampleList decode_chunk(uint32_t chunk_id) const { SampleList decoded; - std::ranges::copy((*serialized_data_.ptr())->iterator(chunk_id), DecodeIteratorSentinel{}, std::back_inserter(decoded)); + std::ranges::copy(serialized_data_->iterator(chunk_id), DecodeIteratorSentinel{}, std::back_inserter(decoded)); return decoded; } + + [[nodiscard]] entrypoint_types::SerializedDataPtr* serialized_data_ptr() noexcept { return serialized_data_memory_.ptr(); } + + void take_serialized_data() { serialized_data_ = std::move(serialized_data_memory_.value()); } + + void unload_open_chunks() { + Unloader unloader{storage_}; + unloader.create_snapshot(unloaded_chunks_); + unloader.unload(); + } + + void load_unloaded_chunks(LabelSetID label_set_id) { + std::vector label_set_ids{label_set_id}; + Loader loader{storage_, label_set_ids, static_cast(label_set_ids.size())}; + loader.load_next(unloaded_chunks_.span()); + loader.load_finalize(); + } }; TEST_F(RangeQuerierWrapperFixture, QuerySerializesMatchingOpenChunk) { @@ -154,18 +248,19 @@ TEST_F(RangeQuerierWrapperFixture, QuerySerializesMatchingOpenChunk) { encoder_.encode(0, 3, 3.0); encoder_.encode(0, 4, 4.0); encoder_.encode(0, 5, 5.0); + auto query = query_for(0, 2, 4); - entrypoint_types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_.ptr()}; + entrypoint_types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_ptr()}; // Act wrapper.query(); - serialized_data_.mark_constructed(); + take_serialized_data(); const auto decoded = decode_chunk(0); // Assert ASSERT_FALSE(wrapper.need_loading()); - ASSERT_NE(nullptr, serialized_data_.get()); - ASSERT_EQ(1U, serialized_data_.get()->get_chunks_view().size()); + ASSERT_NE(nullptr, serialized_data_); + ASSERT_EQ(1U, serialized_data_->get_chunks_view().size()); EXPECT_EQ((SampleList{{1, 1.0}, {2, 2.0}, {3, 3.0}, {4, 4.0}, {5, 5.0}}), decoded); } @@ -173,16 +268,47 @@ TEST_F(RangeQuerierWrapperFixture, QuerySerializesEmptyResultWhenSeriesDoesNotMa // Arrange encoder_.encode(0, 10, 10.0); auto query = query_for(0, 1, 5); - entrypoint_types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_.ptr()}; + entrypoint_types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_ptr()}; // Act wrapper.query(); - serialized_data_.mark_constructed(); + take_serialized_data(); // Assert ASSERT_FALSE(wrapper.need_loading()); - ASSERT_NE(nullptr, serialized_data_.get()); - EXPECT_EQ(0U, serialized_data_.get()->get_chunks_view().size()); + ASSERT_NE(nullptr, serialized_data_); + EXPECT_EQ(0U, serialized_data_->get_chunks_view().size()); +} + +TEST_F(RangeQuerierWrapperFixture, QueryDefersSerializationUntilUnloadedSeriesIsLoaded) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 2.0); + encoder_.encode(0, 3, 3.0); + + unload_open_chunks(); + + auto query = query_for(0, 1, 3); + entrypoint_types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_ptr()}; + + // Act + wrapper.query(); + + const auto need_loading = wrapper.need_loading(); + const auto series_to_load_0 = wrapper.series_to_load().is_set(0); + const auto was_default_before_finalize = serialized_data_memory_.has_default_value(); + + load_unloaded_chunks(0); + wrapper.query_finalize(); + take_serialized_data(); + + // Assert + ASSERT_TRUE(need_loading); + EXPECT_TRUE(series_to_load_0); + EXPECT_TRUE(was_default_before_finalize); + ASSERT_NE(nullptr, serialized_data_); + ASSERT_EQ(1U, serialized_data_->get_chunks_view().size()); + EXPECT_EQ((SampleList{{1, 1.0}, {2, 2.0}, {3, 3.0}}), decode_chunk(0)); } } // namespace diff --git a/pp/entrypoint_types/serialized_data_tests.cpp b/pp/entrypoint_types/serialized_data_tests.cpp index 5dc239851..922b07a7f 100644 --- a/pp/entrypoint_types/serialized_data_tests.cpp +++ b/pp/entrypoint_types/serialized_data_tests.cpp @@ -18,9 +18,10 @@ using series_data::decoder::DecodeIteratorSentinel; using series_data::encoder::Sample; using series_data::encoder::SampleList; using series_data::querier::Querier; +using series_data::serialization::SerializedDataView; using Query = series_data::querier::Query>; -class SerializedDataFixture : public testing::Test { +class SerializedDataGoFixture : public testing::Test { protected: DataStorage storage_; Encoder<> encoder_{storage_}; @@ -33,7 +34,7 @@ class SerializedDataFixture : public testing::Test { } }; -TEST_F(SerializedDataFixture, EmptyQueriedChunkListProducesNoChunks) { +TEST_F(SerializedDataGoFixture, EmptyQueriedChunkListProducesNoChunks) { // Arrange // Act @@ -41,15 +42,17 @@ TEST_F(SerializedDataFixture, EmptyQueriedChunkListProducesNoChunks) { // Assert EXPECT_EQ(0U, data.get_chunks_view().size()); + EXPECT_EQ(SerializedDataView::kNoMoreSeries, data.next().first); } -TEST_F(SerializedDataFixture, RoundTripsQueriedOpenChunk) { +TEST_F(SerializedDataGoFixture, RoundTripsQueriedOpenChunk) { // Arrange encoder_.encode(0, 1, 1.0); encoder_.encode(0, 2, 2.0); encoder_.encode(0, 3, 3.0); encoder_.encode(0, 4, 4.0); encoder_.encode(0, 5, 5.0); + const auto& queried_chunks = querier_.query(Query{.time_interval{.min = 1, .max = 5}, .label_set_ids = {0}}); // Act @@ -63,7 +66,7 @@ TEST_F(SerializedDataFixture, RoundTripsQueriedOpenChunk) { EXPECT_EQ((SampleList{{1, 1.0}, {2, 2.0}, {3, 3.0}, {4, 4.0}, {5, 5.0}}), decoded); } -TEST_F(SerializedDataFixture, RoundTripsQueriedFinalizedChunk) { +TEST_F(SerializedDataGoFixture, RoundTripsQueriedFinalizedChunk) { // Arrange encoder_.encode(0, 1, 1.0); encoder_.encode(0, 2, 2.0); @@ -71,7 +74,9 @@ TEST_F(SerializedDataFixture, RoundTripsQueriedFinalizedChunk) { encoder_.encode(0, 4, 4.0); encoder_.encode(0, 5, 5.0); ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 10, 10.0); + const auto& queried_chunks = querier_.query(Query{.time_interval{.min = 1, .max = 5}, .label_set_ids = {0}}); // Act @@ -85,4 +90,31 @@ TEST_F(SerializedDataFixture, RoundTripsQueriedFinalizedChunk) { EXPECT_EQ((SampleList{{1, 1.0}, {2, 2.0}, {3, 3.0}, {4, 4.0}, {5, 5.0}}), decoded); } +TEST_F(SerializedDataGoFixture, NextReturnsChunkIdsForAllQueriedSeries) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 2.0); + encoder_.encode(0, 3, 3.0); + + encoder_.encode(1, 1, 11.0); + encoder_.encode(1, 2, 12.0); + encoder_.encode(1, 3, 13.0); + + const auto& queried_chunks = querier_.query(Query{.time_interval{.min = 1, .max = 3}, .label_set_ids = {0, 1}}); + + // Act + entrypoint_types::SerializedDataGo data{storage_, queried_chunks}; + const auto series0 = data.next(); + const auto series1 = data.next(); + const auto end = data.next(); + + // Assert + ASSERT_EQ(2U, data.get_chunks_view().size()); + EXPECT_EQ(0U, series0.first); + EXPECT_EQ(1U, series1.first); + EXPECT_EQ(SerializedDataView::kNoMoreSeries, end.first); + EXPECT_EQ((SampleList{{1, 1.0}, {2, 2.0}, {3, 3.0}}), decode_chunk(data, series0.second)); + EXPECT_EQ((SampleList{{1, 11.0}, {2, 12.0}, {3, 13.0}}), decode_chunk(data, series1.second)); +} + } // namespace From 049edea00ffcb9af33964a2f0544bf0f39a8658a Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Tue, 30 Jun 2026 07:26:40 +0000 Subject: [PATCH 3/5] entrypoint types tests build --- pp/BUILD | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pp/BUILD b/pp/BUILD index 1ae8e34da..fbe8bba76 100644 --- a/pp/BUILD +++ b/pp/BUILD @@ -202,7 +202,11 @@ cc_test( name = "entrypoint_types_test", srcs = glob(["entrypoint_types/**/*_tests.cpp"]), deps = [ + ":bare_bones", ":entrypoint_types", + ":metrics", + ":prometheus", + ":wal", "@gtest//:gtest_main", ], ) From 314171383cb40cde6a0a47aa7ff841904887e00f Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Tue, 30 Jun 2026 11:04:39 +0000 Subject: [PATCH 4/5] build fixes --- pp/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pp/BUILD b/pp/BUILD index df792a66b..a7695adcf 100644 --- a/pp/BUILD +++ b/pp/BUILD @@ -200,7 +200,7 @@ cc_library( cc_test( name = "entrypoint_types_test", - srcs = glob(["entrypoint_types/**/*_tests.cpp"]), + srcs = glob(["entrypoint/types/**/*_tests.cpp"]), deps = [ ":bare_bones", ":entrypoint_types", From caa02a2cbc94efc20fae6e0485a879a5eb615b68 Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Tue, 30 Jun 2026 11:19:30 +0000 Subject: [PATCH 5/5] update namespaces in tests --- pp/entrypoint/bridge/go_constants.cpp | 24 ------------------- pp/entrypoint/types/go_constants_tests.cpp | 2 +- pp/entrypoint/types/loader_tests.cpp | 10 ++++---- pp/entrypoint/types/lss_tests.cpp | 24 +++++++++---------- pp/entrypoint/types/querier_tests.cpp | 22 ++++++++--------- pp/entrypoint/types/serialized_data_tests.cpp | 10 ++++---- 6 files changed, 34 insertions(+), 58 deletions(-) delete mode 100644 pp/entrypoint/bridge/go_constants.cpp diff --git a/pp/entrypoint/bridge/go_constants.cpp b/pp/entrypoint/bridge/go_constants.cpp deleted file mode 100644 index d6b50d223..000000000 --- a/pp/entrypoint/bridge/go_constants.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#include "entrypoint/types/go_constants.h" -#include "entrypoint/types/serialized_data.h" -#include "metrics/storage.h" -#include "prometheus/relabeler.h" -#include "wal/output_decoder.h" -#include "wal/segment_samples_storage.h" - -namespace { - -static_assert(sizeof(std::vector) == Sizeof_StdVector); -static_assert(sizeof(BareBones::Vector) == Sizeof_BareBonesVector); -static_assert(sizeof(roaring::Roaring) == Sizeof_RoaringBitset); - -static_assert(sizeof(PromPP::Prometheus::Relabel::InnerSeries) == Sizeof_InnerSeries); - -static_assert(sizeof(entrypoint::types::SerializedDataIterator) == Sizeof_SerializedDataIterator); - -static_assert(sizeof(metrics::Storage::Iterator) == Sizeof_MetricsIterator); - -static_assert(sizeof(PromPP::WAL::SegmentSamplesStorage) == Sizeof_SegmentSamplesStorage); -static_assert(sizeof(PromPP::WAL::ProtobufEncoder) == Sizeof_RemoteWriteMessageEncoder); -static_assert(sizeof(PromPP::WAL::SegmentSamplesStorageList::Iterator) == Sizeof_SegmentSamplesStorageListIterator); - -} // namespace \ No newline at end of file diff --git a/pp/entrypoint/types/go_constants_tests.cpp b/pp/entrypoint/types/go_constants_tests.cpp index 4bd0d9bd6..73d81dc88 100644 --- a/pp/entrypoint/types/go_constants_tests.cpp +++ b/pp/entrypoint/types/go_constants_tests.cpp @@ -19,7 +19,7 @@ TEST(GoConstantsTest, CompileTimeSizesMatchConstants) { static_assert(sizeof(PromPP::Prometheus::Relabel::InnerSeries) == Sizeof_InnerSeries); - static_assert(sizeof(entrypoint_types::SerializedDataIterator) == Sizeof_SerializedDataIterator); + static_assert(sizeof(entrypoint::types::SerializedDataIterator) == Sizeof_SerializedDataIterator); static_assert(sizeof(metrics::Storage::Iterator) == Sizeof_MetricsIterator); diff --git a/pp/entrypoint/types/loader_tests.cpp b/pp/entrypoint/types/loader_tests.cpp index 65f8c0d18..32c4cce5c 100644 --- a/pp/entrypoint/types/loader_tests.cpp +++ b/pp/entrypoint/types/loader_tests.cpp @@ -10,7 +10,7 @@ namespace { -using entrypoint_types::QueryableEncodingBimap; +using entrypoint::types::QueryableEncodingBimap; using PromPP::Primitives::LabelViewSet; using series_data::DataStorage; using series_data::Decoder; @@ -62,7 +62,7 @@ TEST_F(RevertableLoaderFixture, LoadFinalizeRestoresUnloadedOpenChunk) { unloader_.unload(); // Act - entrypoint_types::RevertableLoader loader{storage_, lss_.ls_id_set().begin(), lss_.ls_id_set().end(), 1}; + entrypoint::types::RevertableLoader loader{storage_, lss_.ls_id_set().begin(), lss_.ls_id_set().end(), 1}; loader.load_next(stream_.span()); loader.load_finalize(); @@ -77,7 +77,7 @@ TEST_F(RevertableLoaderFixture, RevertRestoresUnloadedOpenChunk) { unloader_.unload(); const auto trimmed_stream = open_chunk_stream(0); - entrypoint_types::RevertableLoader loader{storage_, lss_.ls_id_set().begin(), lss_.ls_id_set().end(), 1}; + entrypoint::types::RevertableLoader loader{storage_, lss_.ls_id_set().begin(), lss_.ls_id_set().end(), 1}; loader.load_next(stream_.span()); loader.load_finalize(); @@ -98,7 +98,7 @@ TEST_F(RevertableLoaderFixture, LoadFinalizeLoadsSeriesByBatch) { unloader_.create_snapshot(stream_); unloader_.unload(); - entrypoint_types::RevertableLoader loader{storage_, lss_.ls_id_set().begin(), lss_.ls_id_set().end(), 2}; + entrypoint::types::RevertableLoader loader{storage_, lss_.ls_id_set().begin(), lss_.ls_id_set().end(), 2}; // Act loader.load_next(stream_.span()); @@ -134,7 +134,7 @@ TEST_F(RevertableLoaderFixture, RevertRestoresSeriesLoadedAcrossBatches) { const auto trimmed_stream1 = open_chunk_stream(1); const auto trimmed_stream2 = open_chunk_stream(2); - entrypoint_types::RevertableLoader loader{storage_, lss_.ls_id_set().begin(), lss_.ls_id_set().end(), 2}; + entrypoint::types::RevertableLoader loader{storage_, lss_.ls_id_set().begin(), lss_.ls_id_set().end(), 2}; loader.load_next(stream_.span()); loader.load_finalize(); diff --git a/pp/entrypoint/types/lss_tests.cpp b/pp/entrypoint/types/lss_tests.cpp index 4f33e3a64..fde5bd689 100644 --- a/pp/entrypoint/types/lss_tests.cpp +++ b/pp/entrypoint/types/lss_tests.cpp @@ -12,14 +12,14 @@ namespace { -using entrypoint_types::create_lss; -using entrypoint_types::create_snapshot_lss; -using entrypoint_types::EncodingBimap; -using entrypoint_types::LssType; -using entrypoint_types::QueryableEncodingBimap; -using entrypoint_types::ReallocationsDetector; -using entrypoint_types::ShrinkAwareSnapshotLSS; -using entrypoint_types::SnapshotLSS; +using entrypoint::types::create_lss; +using entrypoint::types::create_snapshot_lss; +using entrypoint::types::EncodingBimap; +using entrypoint::types::LssType; +using entrypoint::types::QueryableEncodingBimap; +using entrypoint::types::ReallocationsDetector; +using entrypoint::types::ShrinkAwareSnapshotLSS; +using entrypoint::types::SnapshotLSS; using PromPP::Primitives::LabelViewSet; TEST(LssTest, CreateLssEncodingBimapSelectsExpectedAlternative) { @@ -77,7 +77,7 @@ class SnapshotLssFixture : public testing::Test { const LabelViewSet ls3_{{"job", "d"}}; const LabelViewSet ls4_{{"job", "e"}}; - entrypoint_types::LssVariantPtr create_queryable_lss() const { + entrypoint::types::LssVariantPtr create_queryable_lss() const { auto lss = create_lss(LssType::kQueryableEncodingBimap); auto& bimap = std::get(*lss); @@ -92,14 +92,14 @@ class SnapshotLssFixture : public testing::Test { return lss; } - entrypoint_types::LssVariantPtr create_fixed_lss() const { + entrypoint::types::LssVariantPtr create_fixed_lss() const { auto lss = create_queryable_lss(); std::get(*lss).set_pending_shrink_boundary(kShrinkBoundary); return lss; } - entrypoint_types::LssVariantPtr create_shrunk_lss() const { + entrypoint::types::LssVariantPtr create_shrunk_lss() const { QueryableEncodingBimap seeded_lss; seeded_lss.find_or_emplace(ls0_); seeded_lss.find_or_emplace(ls1_); @@ -152,7 +152,7 @@ TEST_F(SnapshotLssFixture, FromFixedQueryableLssIsShrinkAware) { const auto snapshot = create_snapshot_lss(*lss); // Assert - EXPECT_TRUE(std::holds_alternative(*snapshot)); + EXPECT_TRUE(std::holds_alternative(*snapshot)); } TEST_F(SnapshotLssFixture, ShrinkAwareResolvesSurvivingPreBoundarySeries) { diff --git a/pp/entrypoint/types/querier_tests.cpp b/pp/entrypoint/types/querier_tests.cpp index 898ead327..8ee230f34 100644 --- a/pp/entrypoint/types/querier_tests.cpp +++ b/pp/entrypoint/types/querier_tests.cpp @@ -27,7 +27,7 @@ using series_data::encoder::Sample; using series_data::encoder::SampleList; using series_data::unloading::Loader; using series_data::unloading::Unloader; -using InstantQuerierWrapper = entrypoint_types::InstantQuerierWithArgumentsWrapper, std::span>; +using InstantQuerierWrapper = entrypoint::types::InstantQuerierWithArgumentsWrapper, std::span>; using RangeQuery = series_data::querier::Query>; template @@ -60,7 +60,7 @@ class RangeQuerierUninitializedMemoryFixture : public testing::Test { DataStorage storage_; Encoder<> encoder_{storage_}; BareBones::ShrinkedToFitOStringStream unloaded_chunks_; - UninitializedMemory serialized_data_memory_; + UninitializedMemory serialized_data_memory_; RangeQuery query_for(LabelSetID label_set_id, int64_t min, int64_t max) { Slice label_set_ids; @@ -68,7 +68,7 @@ class RangeQuerierUninitializedMemoryFixture : public testing::Test { return RangeQuery{.time_interval{.min = min, .max = max}, .label_set_ids = std::move(label_set_ids)}; } - [[nodiscard]] entrypoint_types::SerializedDataPtr* serialized_data_ptr() noexcept { return serialized_data_memory_.ptr(); } + [[nodiscard]] entrypoint::types::SerializedDataPtr* serialized_data_ptr() noexcept { return serialized_data_memory_.ptr(); } void unload_open_chunks() { Unloader unloader{storage_}; @@ -89,7 +89,7 @@ TEST_F(RangeQuerierUninitializedMemoryFixture, QueryWritesSerializedDataToPrepar encoder_.encode(0, 1, 1.0); auto query = query_for(0, 1, 1); const auto was_default_before_prepare = serialized_data_memory_.has_default_value(); - entrypoint_types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_ptr()}; + entrypoint::types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_ptr()}; // Act wrapper.query(); @@ -110,7 +110,7 @@ TEST_F(RangeQuerierUninitializedMemoryFixture, QueryFinalizeWritesSerializedData auto query = query_for(0, 1, 3); const auto was_default_before_prepare = serialized_data_memory_.has_default_value(); - entrypoint_types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_ptr()}; + entrypoint::types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_ptr()}; // Act wrapper.query(); @@ -208,8 +208,8 @@ class RangeQuerierWrapperFixture : public testing::Test { DataStorage storage_; Encoder<> encoder_{storage_}; BareBones::ShrinkedToFitOStringStream unloaded_chunks_; - UninitializedMemory serialized_data_memory_; - entrypoint_types::SerializedDataPtr serialized_data_; + UninitializedMemory serialized_data_memory_; + entrypoint::types::SerializedDataPtr serialized_data_; RangeQuery query_for(LabelSetID label_set_id, int64_t min, int64_t max) { Slice label_set_ids; @@ -223,7 +223,7 @@ class RangeQuerierWrapperFixture : public testing::Test { return decoded; } - [[nodiscard]] entrypoint_types::SerializedDataPtr* serialized_data_ptr() noexcept { return serialized_data_memory_.ptr(); } + [[nodiscard]] entrypoint::types::SerializedDataPtr* serialized_data_ptr() noexcept { return serialized_data_memory_.ptr(); } void take_serialized_data() { serialized_data_ = std::move(serialized_data_memory_.value()); } @@ -250,7 +250,7 @@ TEST_F(RangeQuerierWrapperFixture, QuerySerializesMatchingOpenChunk) { encoder_.encode(0, 5, 5.0); auto query = query_for(0, 2, 4); - entrypoint_types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_ptr()}; + entrypoint::types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_ptr()}; // Act wrapper.query(); @@ -268,7 +268,7 @@ TEST_F(RangeQuerierWrapperFixture, QuerySerializesEmptyResultWhenSeriesDoesNotMa // Arrange encoder_.encode(0, 10, 10.0); auto query = query_for(0, 1, 5); - entrypoint_types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_ptr()}; + entrypoint::types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_ptr()}; // Act wrapper.query(); @@ -289,7 +289,7 @@ TEST_F(RangeQuerierWrapperFixture, QueryDefersSerializationUntilUnloadedSeriesIs unload_open_chunks(); auto query = query_for(0, 1, 3); - entrypoint_types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_ptr()}; + entrypoint::types::RangeQuerierWithArgumentsWrapperV2 wrapper{storage_, query, serialized_data_ptr()}; // Act wrapper.query(); diff --git a/pp/entrypoint/types/serialized_data_tests.cpp b/pp/entrypoint/types/serialized_data_tests.cpp index 442eaaf92..5ace44325 100644 --- a/pp/entrypoint/types/serialized_data_tests.cpp +++ b/pp/entrypoint/types/serialized_data_tests.cpp @@ -27,7 +27,7 @@ class SerializedDataGoFixture : public testing::Test { Encoder<> encoder_{storage_}; Querier querier_{storage_}; - [[nodiscard]] static SampleList decode_chunk(const entrypoint_types::SerializedDataGo& data, uint32_t chunk_id) { + [[nodiscard]] static SampleList decode_chunk(const entrypoint::types::SerializedDataGo& data, uint32_t chunk_id) { SampleList decoded; std::ranges::copy(data.iterator(chunk_id), DecodeIteratorSentinel{}, std::back_inserter(decoded)); return decoded; @@ -38,7 +38,7 @@ TEST_F(SerializedDataGoFixture, EmptyQueriedChunkListProducesNoChunks) { // Arrange // Act - entrypoint_types::SerializedDataGo data{storage_, {}}; + entrypoint::types::SerializedDataGo data{storage_, {}}; // Assert EXPECT_EQ(0U, data.get_chunks_view().size()); @@ -56,7 +56,7 @@ TEST_F(SerializedDataGoFixture, RoundTripsQueriedOpenChunk) { const auto& queried_chunks = querier_.query(Query{.time_interval{.min = 1, .max = 5}, .label_set_ids = {0}}); // Act - entrypoint_types::SerializedDataGo data{storage_, queried_chunks}; + entrypoint::types::SerializedDataGo data{storage_, queried_chunks}; const auto next_series = data.next(); const auto decoded = decode_chunk(data, 0); @@ -80,7 +80,7 @@ TEST_F(SerializedDataGoFixture, RoundTripsQueriedFinalizedChunk) { const auto& queried_chunks = querier_.query(Query{.time_interval{.min = 1, .max = 5}, .label_set_ids = {0}}); // Act - entrypoint_types::SerializedDataGo data{storage_, queried_chunks}; + entrypoint::types::SerializedDataGo data{storage_, queried_chunks}; const auto next_series = data.next(); const auto decoded = decode_chunk(data, 0); @@ -103,7 +103,7 @@ TEST_F(SerializedDataGoFixture, NextReturnsChunkIdsForAllQueriedSeries) { const auto& queried_chunks = querier_.query(Query{.time_interval{.min = 1, .max = 3}, .label_set_ids = {0, 1}}); // Act - entrypoint_types::SerializedDataGo data{storage_, queried_chunks}; + entrypoint::types::SerializedDataGo data{storage_, queried_chunks}; const auto series0 = data.next(); const auto series1 = data.next(); const auto end = data.next();