Skip to content

Refine index size validation #447

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
williswus wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Refine index size validation #447

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d80c179
Refine index size validation
williswus Aug 20, 2026
277deed
test: cover oversized LSMT index merge
williswus Aug 21, 2026
fed7331
lsmt: centralize merge exception handling
williswus Sep 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/overlaybd/lsmt/file.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1338,6 +1338,11 @@ static HeaderTrailer *verify_ht(IFile *file, char *buf, bool is_trailer, ssize_t

if (!pht->verify_magic() || !pht->is_header())
LOG_ERROR_RETURN(0, nullptr, "header magic/type don't match");

if (pht->index_size > MAX_LSMT_RO_INDEX_SIZE)
LOG_ERROR_RETURN(0, nullptr, "LSMT RO index size ` exceeds maximum `",
pht->index_size + 0, MAX_LSMT_RO_INDEX_SIZE);

return pht;
}
if (st_size == -1) {
Expand Down
71 changes: 38 additions & 33 deletions src/overlaybd/lsmt/index.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include <set>
#include <algorithm>
#include <memory>
#include <stdexcept>
#include <photon/common/alog.h>
#include <photon/fs/filesystem.h>
#include <photon/common/utility.h>
Expand Down Expand Up @@ -622,10 +623,9 @@ class Index0 : public IComboIndex {
UNIMPLEMENTED(int commit_index0() override);
};

static bool merge_indexes(uint8_t level, vector<SegmentMapping> &mapping, const Index **pindexes,
static void merge_indexes(uint8_t level, vector<SegmentMapping> &mapping, const Index **pindexes,
std::size_t n, uint64_t begin, uint64_t end, bool change_tag = true,
size_t max_level = 0,
size_t max_index_size = MAX_LSMT_INDEX_SIZE);
size_t max_level = 0);

class ComboIndex : public Index0 {
public:
Expand Down Expand Up @@ -741,8 +741,11 @@ class ComboIndex : public Index0 {
virtual Index *rebuild_backing_index(Index *highlevel_idx, size_t max_level) {
vector<SegmentMapping> mappings;
const Index *indexes[2] = {highlevel_idx, const_cast<Index *>(m_backing_index)};
if (!merge_indexes(0, mappings, indexes, 2, 0, UINT64_MAX, false, max_level))
try {
merge_indexes(0, mappings, indexes, 2, 0, UINT64_MAX, false, max_level);
} catch (const std::length_error &) {
return nullptr;
}
return new Index(std::move(mappings));
}

Expand All @@ -755,7 +758,9 @@ class ComboIndex : public Index0 {
return ro_idx0;
}
const Index *indexes[2] = {ro_idx0, const_cast<Index *>(m_backing_index)};
if (!merge_indexes(0, mappings, indexes, 2, 0, UINT64_MAX, false, 2)) {
try {
merge_indexes(0, mappings, indexes, 2, 0, UINT64_MAX, false, 2);
} catch (const std::length_error &) {
delete ro_idx0;
return nullptr;
}
Expand Down Expand Up @@ -844,68 +849,65 @@ IMemoryIndex *create_level_index(const SegmentMapping *pmappings, size_t n, uint
return (ok1 && ok2) ? new LevelIndex(pmappings, n, copy_mode) : nullptr;
}

static bool merge_indexes(uint8_t level, vector<SegmentMapping> &mapping, const Index **pindexes,
static void merge_indexes(uint8_t level, vector<SegmentMapping> &mapping, const Index **pindexes,

@lihuiba lihuiba Aug 26, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add a wrapper function, merge_indexes_catch, that catches exception(s) and translate it to return a bool value. Users use the wrapper function only.

size_t n, uint64_t begin, uint64_t end, bool change_tag,
size_t max_level, size_t max_index_size) {
size_t max_level) {

if (pindexes == nullptr)
return true;
return;
if (change_tag) {
if (n == 0)
return true;
return;
} else {
if (max_level == 0)
return true;
return;
}
if (begin >= end)
return true;
return;

auto begin0 = begin;
auto size0 = mapping.size();
auto pi0 = pindexes[0];
for (auto it = pi0->lower_bound(begin); it != pi0->end() && it->offset < end; ++it) {
if (it->offset > begin) {
if (change_tag) {
if (!merge_indexes(level + 1, mapping, pindexes + 1, n - 1, begin, it->offset,
true, max_level, max_index_size))
return false;
} else {
if (change_tag)
merge_indexes(level + 1, mapping, pindexes + 1, n - 1, begin, it->offset);
else {
int k = (n <= 1 ? 0 : 1);
if (!merge_indexes(level + 1, mapping, pindexes + k, 0, begin, it->offset, false,
max_level - 1, max_index_size))
return false;
merge_indexes(level + 1, mapping, pindexes + k, 0, begin, it->offset, false,
max_level - 1);
}
}
if (mapping.size() >= max_index_size)
LOG_ERROR_RETURN(0, false, "Merged LSMT index size ` exceeds maximum `",
mapping.size() + 1, max_index_size);

if (mapping.size() >= MAX_LSMT_INDEX_SIZE) {
LOG_ERROR("Merged LSMT index size ` exceeds maximum `", mapping.size() + 1,
MAX_LSMT_INDEX_SIZE);
throw std::length_error("Merged LSMT index size exceeds maximum");
}
Comment on lines +881 to +885

mapping.push_back(*it);
if (change_tag) {
mapping.back().tag = level;
}
begin = it->end();
}

if (begin < end) {
if (change_tag) {
if (!merge_indexes(level + 1, mapping, pindexes + 1, n - 1, begin, end,
true, max_level, max_index_size))
return false;
} else {
if (change_tag)
merge_indexes(level + 1, mapping, pindexes + 1, n - 1, begin, end);
else {
int k = (n <= 1 ? 0 : 1);
if (!merge_indexes(level + 1, mapping, pindexes + k, 0, begin, end, false,
max_level - 1, max_index_size))
return false;
merge_indexes(level + 1, mapping, pindexes + k, 0, begin, end, false,
max_level - 1);
}
}

if (mapping.size() > size0) {
if (mapping[size0].offset < begin0)
mapping[size0].forward_offset_to(begin0);
if (mapping.back().end() > end)
mapping.back().backward_end_to(end);
}

return true;
}

IComboIndex *create_combo_index(IMemoryIndex0 *index0, const IMemoryIndex *index,
Expand Down Expand Up @@ -969,8 +971,11 @@ IMemoryIndex *merge_memory_indexes(const IMemoryIndex **pindexes, size_t n) {
vector<SegmentMapping> mapping;
auto pi = (const Index **)pindexes;
mapping.reserve(pi[0]->size());
if (!merge_indexes(0, mapping, pi, n, 0, UINT64_MAX))
try {
merge_indexes(0, mapping, pi, n, 0, UINT64_MAX);
} catch (const std::length_error &) {
return nullptr;
}

if (pindexes[0]->vsize() < static_cast<uint64_t>(UINT32_MAX) * ALIGNMENT
&& mapping.size() < NODES_PER_LEVEL_32[MAX_LEVEL_32-1]) {
Expand Down
14 changes: 0 additions & 14 deletions src/overlaybd/lsmt/test/test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,20 +314,6 @@ inline void test_merge_combo(const IMemoryIndex *indexes[], size_t ni, // num of
test_combo(indexes, ni, stdrst, NR);
}

TEST(Index, reject_oversized_merge) {
SegmentMapping mapping0[] = {{0, 1, 0}, {2, 1, 2}};
SegmentMapping mapping1[] = {{1, 1, 1}, {3, 1, 3}};

Index index0(mapping0, LEN(mapping0), false);
Index index1(mapping1, LEN(mapping1), false);
const Index *indexes[] = {&index0, &index1};

vector<SegmentMapping> merged;
EXPECT_FALSE(merge_indexes(0, merged, indexes, LEN(indexes), 0, UINT64_MAX,
true, 0, 3));
EXPECT_EQ(merged.size(), 3);
}

TEST(Index, merge) {
const static SegmentMapping mapping0[] = {{5, 5, 0}, {10, 10, 50}, {100, 10, 20}};
Comment on lines -310 to 337
const static SegmentMapping mapping1[] = {{0, 1, 7}, {2, 4, 5}, {15, 10, 22},
Expand Down
9 changes: 5 additions & 4 deletions src/overlaybd/zfile/zfile.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1069,6 +1069,11 @@ bool load_jump_table(IFile *file, CompressionFile::HeaderTrailer *pheader_traile
if (pht->is_valid() == false) {
LOG_ERROR_RETURN(0, false, "digest verification failed.");
}

if (pht->index_size > MAX_ZFILE_INDEX_SIZE)
LOG_ERROR_RETURN(0, false, "ZFile index size ` exceeds maximum `",
pht->index_size + 0, MAX_ZFILE_INDEX_SIZE);

struct stat stat;
ret = file->fstat(&stat);
if (ret < 0) {
Expand Down Expand Up @@ -1108,10 +1113,6 @@ bool load_jump_table(IFile *file, CompressionFile::HeaderTrailer *pheader_traile
if (index_bytes > trailer_offset - pht->index_offset)
LOG_ERROR_RETURN(0, false, "invalid index bytes or size. ");
} else {
if (pht->index_size > MAX_ZFILE_INDEX_SIZE)
LOG_ERROR_RETURN(0, false, "ZFile index size ` exceeds maximum `",
pht->index_size + 0, MAX_ZFILE_INDEX_SIZE);

index_bytes = pht->index_size * sizeof(uint32_t);
LOG_INFO("read overwrite header. idx_offset: `, idx_bytes: `, dict_size: `, use_dict: `",
pht->index_offset, index_bytes, pht->opt.dict_size, pht->opt.use_dict);
Expand Down
Loading