diff --git a/builds/gnu/NEWS b/builds/gnu/NEWS index e1873049f..f9d5bbadd 100644 --- a/builds/gnu/NEWS +++ b/builds/gnu/NEWS @@ -1 +1 @@ -See https://libbitcoin.org \ No newline at end of file +See https://libbitcoin.info \ No newline at end of file diff --git a/include/bitcoin/database/impl/memory/mmap.ipp b/include/bitcoin/database/impl/memory/mmap.ipp index 27cdf1b92..5822e1136 100644 --- a/include/bitcoin/database/impl/memory/mmap.ipp +++ b/include/bitcoin/database/impl/memory/mmap.ipp @@ -39,6 +39,7 @@ CLASS::mmap(const path& filename, const storage_settings& settings, : filenames_{ filename }, minimum_(to_rows(settings.size)), expansion_(settings.rate), + headroom_(system::possible_narrow_cast(settings.headroom)), access_(settings.access), random_(random), staged_(staged), @@ -53,6 +54,7 @@ CLASS::mmap(const paths& filenames, const storage_settings& settings, : filenames_(filenames), minimum_(to_rows(settings.size)), expansion_(settings.rate), + headroom_(system::possible_narrow_cast(settings.headroom)), access_(settings.access), random_(random), staged_(staged), diff --git a/include/bitcoin/database/impl/memory/mmap_private.ipp b/include/bitcoin/database/impl/memory/mmap_private.ipp index 3fbe4fd24..8d4046723 100644 --- a/include/bitcoin/database/impl/memory/mmap_private.ipp +++ b/include/bitcoin/database/impl/memory/mmap_private.ipp @@ -107,11 +107,32 @@ bool CLASS::unmap_all_(std::index_sequence) NOEXCEPT TEMPLATE template -bool CLASS::remap_all_(size_t capacity, std::index_sequence) NOEXCEPT +bool CLASS::remap_all_(size_t capacity, std::index_sequence, + bool final) NOEXCEPT { - if (!(remap_(capacity) && ...)) + // Probe the wave's disk requirement before touching any column file: a + // refused wave then retains no surplus provisioning (columns cannot be + // trimmed after a partial wave, as msc cannot shrink a mapped file). + if (!probe_(capacity)) { - capacity_.store(zero); + if (final) + { + using namespace system; + set_disk_space(ceilinged_add(headroom_, ceilinged_multiply( + floored_subtract(capacity, file_.load()), stride))); + } + + return false; + } + + if (!(remap_(capacity, final) && ...)) + { + // A non-final refusal leaves the maps and capacity intact for the + // caller's reduced retry (columns already grown by the refused + // attempt harmlessly retain surplus commitment or provisioning). + if (final) + capacity_.store(zero); + return false; } @@ -276,7 +297,7 @@ bool CLASS::map_() NOEXCEPT // Remapping has no effect on logical size, sets map_/capacity_. TEMPLATE template -bool CLASS::remap_(size_t size) NOEXCEPT +bool CLASS::remap_(size_t size, bool final) NOEXCEPT { BC_ASSERT(size >= logical_.load()); @@ -288,12 +309,12 @@ bool CLASS::remap_(size_t size) NOEXCEPT // The file is preallocated to capacity, preserving disk full detection at // allocation, and growth commits reserved anonymous pages in place, so no // mapping is released and the map base is stable within the reservation. - if (!resize_(size)) + if (!resize_(size, final)) return false; - return commit_(size); + return commit_(size, final); #else - if (!resize_(size)) + if (!resize_(size, final)) return false; #if defined(HAVE_MSC) @@ -315,7 +336,7 @@ bool CLASS::remap_(size_t size) NOEXCEPT // disk_full: space is set but no code is set with false return. TEMPLATE template -bool CLASS::resize_(size_t size) NOEXCEPT +bool CLASS::resize_(size_t size, bool final) NOEXCEPT { // The file is provisioned ahead of commitment, so growth within the // provisioned extent requires no disk operation (the space is reserved). @@ -323,10 +344,12 @@ bool CLASS::resize_(size_t size) NOEXCEPT if (size <= extent) return true; + using namespace system; const auto target = to_width(size); const auto capacity = to_width(extent); - // Disk full detection, any other failure is an abort. + // Disk full detection, any other failure is an abort. The wave probe + // (remap_all_) precedes, so refusal here is a raced foreign consumer. #if !defined(WITHOUT_FALLOCATE) if (::fallocate(opened_[Column], 0, capacity, target - capacity) == fail) #else @@ -334,11 +357,14 @@ bool CLASS::resize_(size_t size) NOEXCEPT #endif { // Disk full is the only restartable store failure (leave mapped). + // A non-final refusal is not published: the caller retries reduced. + // The published requirement includes the headroom (a retry probes). if (errno == ENOSPC) { - using namespace system; - set_disk_space(ceilinged_multiply(floored_subtract(size, extent), - stride)); + if (final) + set_disk_space(ceilinged_add(headroom_, ceilinged_multiply( + floored_subtract(size, extent), stride))); + return false; } @@ -350,6 +376,42 @@ bool CLASS::resize_(size_t size) NOEXCEPT return true; } +// The wave probe reserves the whole extension plus headroom on the store +// volume (column widths sum to the stride), released upon the grant: a +// refused wave touches no column file, and a granted one leaves the +// headroom unclaimed. +TEMPLATE +bool CLASS::probe_(size_t capacity) NOEXCEPT +{ + using namespace system; + const auto bytes = ceilinged_multiply( + floored_subtract(capacity, file_.load()), stride); + + if (is_zero(bytes)) + return true; + + auto name = filenames_.front(); + name += ".probe"; + auto probe = file::invalid; + if (file::create_file(name)) + probe = file::open(name); + + const auto reserve = ceilinged_add(bytes, headroom_); +#if !defined(WITHOUT_FALLOCATE) + const auto held = (probe != file::invalid) && + (::fallocate(probe, 0, zero, reserve) != fail); +#else + const auto held = (probe != file::invalid) && + (::ftruncate(probe, reserve) != fail); +#endif + + if (probe != file::invalid) + file::close(probe); + + file::remove(name); + return held; +} + // Finalize failure results in unmapped. TEMPLATE template diff --git a/include/bitcoin/database/impl/memory/mmap_staging.ipp b/include/bitcoin/database/impl/memory/mmap_staging.ipp index 3909ef6a9..d6b178a36 100644 --- a/include/bitcoin/database/impl/memory/mmap_staging.ipp +++ b/include/bitcoin/database/impl/memory/mmap_staging.ipp @@ -160,35 +160,42 @@ size_t CLASS::frontier() const NOEXCEPT #if defined(MANAGE_STAGING) +// Claim and record an extent under one lock: a claim never exists outside +// the ring and the ring is start-ordered, so the frontier can never pass an +// unwritten extent (claim-then-record raced the frontier past the claim). +// Returns eof (unclaimed) on insufficient capacity, fault, or disk full. TEMPLATE -void CLASS::record_(size_t start, size_t count) NOEXCEPT +size_t CLASS::record_(size_t count) NOEXCEPT { - if (!staged_ || is_zero(count)) - return; - std::unique_lock extent_lock(extent_mutex_); + if (is_zero(count)) + return logical_.load(); + maintain_(); using namespace system; auto [head, size] = unpack_word(window_.load(relaxed)); // A full ring waits on completions (extents are allocation-coarse, so - // saturation implies extreme concurrency). An unrecorded extent would be - // unsafe: an emptied ring advances the frontier to logical, so untracked - // incomplete writes could settle. Completions are lock-free, so waiting - // needs only this thread's own maintenance; fault or disk full releases - // the wait (recording is then moot, as recovery discards the ring). + // saturation implies extreme concurrency). Completions are lock-free, so + // waiting needs only this thread's own maintenance; fault or disk full + // releases the wait unclaimed (the write then fails fast). while (size == extents) { if (fault_.load() || !is_zero(space_.load())) - return; + return storage::eof; std::this_thread::yield(); maintain_(); std::tie(head, size) = unpack_word(window_.load(relaxed)); } + const auto start = logical_.load(); + if (is_add_overflow(start, count) || + ((start + count) > capacity_.load())) + return storage::eof; + auto& record = ring_.at((head + size) % extents); const auto generation = bit_and(add1(shift_right( record.state.load(relaxed), generation_shift)), generation_mask); @@ -206,6 +213,10 @@ void CLASS::record_(size_t start, size_t count) NOEXCEPT window_.store(pack_word(head, add1(size)), release); if (is_zero(size)) frontier_.store(start); + + logical_.store(start + count); + check_invariants_(); + return start; } // Pop completed extents from the head, advancing the frontier (locked). @@ -397,7 +408,7 @@ bool CLASS::stage_() NOEXCEPT const auto settled = page_floor(to_width(settled_.load())); if ((target > settled) && (mmap_commit(std::next(memory_map_[Column], - settled), target - settled) == fail)) + settled), target - settled, headroom_) == fail)) { teardown_(error::mmap_failure); return false; @@ -440,13 +451,17 @@ bool CLASS::stage_() NOEXCEPT return true; } -// Commit failure results in unmapped. +// Commit failure results in unmapped when final (the default); a non-final +// refusal (in-reservation commit, replacement reservation, or replacement +// commit) returns false with the standing mapping untouched, so the caller +// may iterate a reduced request (admission is evaluated per request, so a +// refused amortization step does not imply exhaustion). // Growth within the reservation commits pages in place (stable map base); an // exhausted reservation is replaced and its unsettled content copied, under // the exclusive remap lock held by the caller. TEMPLATE template -bool CLASS::commit_(size_t size) NOEXCEPT +bool CLASS::commit_(size_t size, bool final) NOEXCEPT { const auto target = to_width(size); @@ -476,9 +491,11 @@ bool CLASS::commit_(size_t size) NOEXCEPT const auto from = std::max(settled, current); if ((target > from) && (mmap_commit(std::next(memory_map_[Column], from), - target - from) == fail)) + target - from, headroom_) == fail)) { - teardown_(error::mmap_failure); + if (final) + teardown_(error::mmap_failure); + return false; } @@ -496,7 +513,9 @@ bool CLASS::commit_(size_t size) NOEXCEPT if (replace == MAP_FAILED) { - teardown_(error::mmap_failure); + if (final) + teardown_(error::mmap_failure); + return false; } @@ -530,10 +549,19 @@ bool CLASS::commit_(size_t size) NOEXCEPT return true; } - if (mmap_commit(std::next(base, settled), target - settled) == fail) + // The replacement commit spans the unsettled prefix, transiently charged + // over the standing reservation, so its refusal is the largest single + // admission of the design: non-final refusal leaves the standing mapping + // untouched for the caller's reduced retry (a reduced ask fits within + // the standing reservation), and settle drainage shrinks the span, so a + // necessity refusal pauses recoverable rather than tearing down. + if (mmap_commit(std::next(base, settled), target - settled, + headroom_) == fail) { ::munmap(replace, reserved); - teardown_(error::mmap_failure); + if (final) + teardown_(error::mmap_failure); + return false; } diff --git a/include/bitcoin/database/impl/memory/mmap_storage.ipp b/include/bitcoin/database/impl/memory/mmap_storage.ipp index dfbee63c8..d757cd0a3 100644 --- a/include/bitcoin/database/impl/memory/mmap_storage.ipp +++ b/include/bitcoin/database/impl/memory/mmap_storage.ipp @@ -186,6 +186,7 @@ code CLASS::reload() NOEXCEPT return error::success; } + // Locked by reader(s), as write suspension is a precondition. return error::reload_locked; } @@ -462,6 +463,42 @@ bool CLASS::truncate(size_t count) NOEXCEPT return true; } +// Iterated growth (callers hold the remap lock). Growth asks are amortized +// (rate surplus over the necessity), and each is admitted only while it +// leaves the configured headroom of the backing resource unclaimed (probed +// with the ask, released on grant), so exhaustion never consumes the +// system's final bytes. A large amortization step can be refused while the +// necessity fits, so iterate: halve the refused surplus toward the +// necessity. Refusal of the necessity is exhaustion, not store damage: +// published as disk full (space set, store intact, writes fail fast until +// cleared), it clears by settle drainage or operator relief, where teardown +// would convert a shortage into a restore. +TEMPLATE +bool CLASS::grow_(size_t end) NOEXCEPT +{ + if (!is_zero(space_.load())) + return false; + + using namespace system; + for (auto extended = to_growth(end); + !remap_all_(extended, sequence{}, false); + extended = ceilinged_add(end, + to_half(floored_subtract(extended, end)))) + { + if (fault_.load()) + return false; + + if (extended <= end) + { + set_disk_space(ceilinged_add(headroom_, ceilinged_multiply( + floored_subtract(end, capacity_.load()), stride))); + return false; + } + } + + return true; +} + TEMPLATE bool CLASS::expand(size_t count) NOEXCEPT { @@ -475,10 +512,8 @@ bool CLASS::expand(size_t count) NOEXCEPT if (count > capacity_.load()) { - const auto extended = to_growth(count); std::unique_lock remap_lock(remap_mutex_); - - if (!remap_all_(extended, sequence{})) + if (!grow_(count)) return false; } @@ -503,10 +538,8 @@ bool CLASS::reserve(size_t count) NOEXCEPT const auto end = logical_.load() + count; if (end > capacity_.load()) { - const auto extended = to_growth(end); std::unique_lock remap_lock(remap_mutex_); - - if (!remap_all_(extended, sequence{})) + if (!grow_(end)) return false; } @@ -522,14 +555,46 @@ bool CLASS::reserve(size_t count) NOEXCEPT TEMPLATE size_t CLASS::allocate(size_t count) NOEXCEPT { + using namespace system; + #if defined(MANAGE_STAGING) // Nothing is held here, so parking cannot deadlock (as with remap waits). throttle_(); -#endif + + // Staged claims serialize with extent recording (record_ locks on every + // claim regardless, so this adds no contention): a claim never exists + // outside the ring, so the frontier can never pass an unwritten extent. + if (staged_) + { + while (true) + { + if (fault_.load() || !loaded_.load()) + return storage::eof; + + const auto start = record_(count); + if (start != storage::eof) + return start; + + // Slow path: serialize capacity growth (at most one grower). + std::unique_lock field_lock(field_mutex_); + + const auto logical = logical_.load(); + if (is_add_overflow(logical, count)) + return storage::eof; + + const auto end = logical + count; + if (end > capacity_.load()) + { + std::unique_lock remap_lock(remap_mutex_); + if (!grow_(end)) + return storage::eof; + } + } + } +#endif // MANAGE_STAGING // Fast path: claim rows within published capacity (no locks). A failed // exchange implies another claim succeeded, so every retry is progress. - using namespace system; auto start = logical_.load(); while (true) { @@ -540,12 +605,7 @@ size_t CLASS::allocate(size_t count) NOEXCEPT break; if (logical_.compare_exchange_weak(start, start + count)) - { -#if defined(MANAGE_STAGING) - record_(start, count); -#endif return start; - } } // Slow path: serialize capacity growth (at most one grower). Fast paths @@ -562,23 +622,16 @@ size_t CLASS::allocate(size_t count) NOEXCEPT if (end <= capacity_.load()) { if (logical_.compare_exchange_weak(start, end)) - { -#if defined(MANAGE_STAGING) - record_(start, count); -#endif return start; - } continue; } - const auto extended = to_growth(end); - // TODO: Could loop over a try lock here and log deadlock warning. std::unique_lock remap_lock(remap_mutex_); // Disk full condition leaves store in valid state despite eof return. - if (!remap_all_(extended, sequence{})) + if (!grow_(end)) return storage::eof; } } diff --git a/include/bitcoin/database/memory/mmap.hpp b/include/bitcoin/database/memory/mmap.hpp index 6dd05f156..df161830f 100644 --- a/include/bitcoin/database/memory/mmap.hpp +++ b/include/bitcoin/database/memory/mmap.hpp @@ -1,4 +1,4 @@ -/** +/** * Copyright (c) 2011-2026 libbitcoin developers * * This file is part of libbitcoin. @@ -280,7 +280,10 @@ class mmap template bool unmap_all_(std::index_sequence) NOEXCEPT; template - bool remap_all_(size_t capacity, std::index_sequence) NOEXCEPT; + bool remap_all_(size_t capacity, std::index_sequence, + bool final=true) NOEXCEPT; + bool grow_(size_t end) NOEXCEPT; + bool probe_(size_t capacity) NOEXCEPT; // mman wrappers, not thread safe. template @@ -292,9 +295,9 @@ class mmap template bool unmap_(size_t size) NOEXCEPT; template - bool remap_(size_t size) NOEXCEPT; + bool remap_(size_t size, bool final=true) NOEXCEPT; template - bool resize_(size_t size) NOEXCEPT; + bool resize_(size_t size, bool final=true) NOEXCEPT; template bool finalize_(size_t size) NOEXCEPT; @@ -312,7 +315,7 @@ class mmap template bool stage_() NOEXCEPT; template - bool commit_(size_t size) NOEXCEPT; + bool commit_(size_t size, bool final=true) NOEXCEPT; template bool settle_(size_t from, size_t to) NOEXCEPT; template @@ -324,7 +327,7 @@ class mmap // staging utilities, not thread safe (claim_ is lock-free thread safe). struct extent; - void record_(size_t start, size_t count) NOEXCEPT; + size_t record_(size_t count) NOEXCEPT; bool claim_(extent& record, size_t count) NOEXCEPT; void maintain_() NOEXCEPT; void discard_() NOEXCEPT; @@ -382,6 +385,7 @@ class mmap const paths filenames_; const size_t minimum_; const size_t expansion_; + const size_t headroom_; const advice access_; const bool random_; const bool staged_; diff --git a/include/bitcoin/database/memory/mstage.hpp b/include/bitcoin/database/memory/mstage.hpp index 43a8b6f2c..6944fd28a 100644 --- a/include/bitcoin/database/memory/mstage.hpp +++ b/include/bitcoin/database/memory/mstage.hpp @@ -42,7 +42,7 @@ void* mmap_reserve(size_t size) NOEXCEPT; /// Commit reserved pages as readable/writable anonymous memory. -int mmap_commit(void* address, size_t size) NOEXCEPT; +int mmap_commit(void* address, size_t size, size_t headroom) NOEXCEPT; /// Replace committed pages with a read-only shared mapping of the file. int mmap_settle(void* address, size_t size, int fd, size_t offset) NOEXCEPT; diff --git a/include/bitcoin/database/memory/settings.hpp b/include/bitcoin/database/memory/settings.hpp index 31880963d..5ba490ece 100644 --- a/include/bitcoin/database/memory/settings.hpp +++ b/include/bitcoin/database/memory/settings.hpp @@ -56,6 +56,10 @@ struct storage_settings /// Body expansion rate (percentage). uint16_t rate{ 5 }; + /// Headroom (bytes): growth is admitted only while it leaves this much + /// of the backing resource (memory commitment, disk space) unclaimed. + uint64_t headroom{ system::power2(28u) }; + /// Page advice for mapped reads (see advice). Bodies default to /// scattered: they are far too large to reside, and validation reads /// prevouts from arbitrary earlier blocks, so read-ahead manufactures diff --git a/include/bitcoin/database/store.hpp b/include/bitcoin/database/store.hpp index 3b08b020a..541cc47fa 100644 --- a/include/bitcoin/database/store.hpp +++ b/include/bitcoin/database/store.hpp @@ -237,7 +237,8 @@ class store // Heads are minimally allocated with no expansion (heads size to their // configured buckets at table creation) and randomly probed (the advice // preserves optimal classic mapped reads; staged heads are anonymous). - static constexpr storage_settings head_settings{ 1, 0, advice::random }; + static constexpr storage_settings head_settings{ .size = 1, .rate = 0, + .access = advice::random }; // Bodies are append-only, so stage writes in anonymous memory where the // staging backend is built (heads update in place and remain resident). diff --git a/src/memory/mstage.cpp b/src/memory/mstage.cpp index d11e950df..45e787f4d 100644 --- a/src/memory/mstage.cpp +++ b/src/memory/mstage.cpp @@ -31,6 +31,8 @@ #if defined(HAVE_APPLE) #include #include + #include + #include #endif #if defined(HAVE_LINUX) #include @@ -46,9 +48,69 @@ void* mmap_reserve(size_t size) NOEXCEPT -1, 0); } -int mmap_commit(void* address, size_t size) NOEXCEPT +#if defined(HAVE_APPLE) + +// Darwin admits every anonymous ask (exhaustion arrives at first touch), so +// admission is computed: free ram plus swap growth room (darwin swap is +// files created on demand in the vm volume). A failed measurement term +// contributes zero (conservative), and the bound moves at disk speed, so +// the headroom absorbs the measure-to-touch race. +static bool mmap_admit(size_t size) NOEXCEPT { - return ::mprotect(address, size, PROT_READ | PROT_WRITE); + vm_statistics64_data_t stats{}; + mach_msg_type_number_t count = HOST_VM_INFO64_COUNT; + uint64_t ram{}; + if (::host_statistics64(::mach_host_self(), HOST_VM_INFO64, + reinterpret_cast(&stats), &count) == KERN_SUCCESS) + ram = ceilinged_multiply( + ceilinged_add(stats.free_count, stats.inactive_count), + possible_wide_cast(vm_page_size)); + + xsw_usage swap{}; + size_t length = sizeof(swap); + uint64_t slack{}; + if (::sysctlbyname("vm.swapusage", &swap, &length, nullptr, 0) == 0) + slack = floored_subtract(swap.xsu_total, swap.xsu_used); + + struct statfs volume{}; + uint64_t growth{}; + if (::statfs("/System/Volumes/VM", &volume) == 0) + growth = ceilinged_multiply(possible_wide_cast( + volume.f_bavail), possible_wide_cast(volume.f_bsize)); + + return possible_wide_cast(size) <= + ceilinged_add(ram, ceilinged_add(slack, growth)); +} + +#endif // HAVE_APPLE + +// The commitment is granted only if it leaves the headroom free: the probe +// charges the headroom alongside the request (atomic with its admission) +// and releases it upon the grant. Darwin refuses no charge, so its +// admission is computed rather than delegated. +int mmap_commit(void* address, size_t size, size_t headroom) NOEXCEPT +{ + if (is_zero(headroom)) + return ::mprotect(address, size, PROT_READ | PROT_WRITE); + +#if defined(HAVE_APPLE) + if (!mmap_admit(ceilinged_add(size, headroom))) + return -1; +#else + const auto probe = ::mmap(nullptr, headroom, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + + if (probe == MAP_FAILED) + return -1; +#endif + + const auto result = ::mprotect(address, size, PROT_READ | PROT_WRITE); + +#if !defined(HAVE_APPLE) + ::munmap(probe, headroom); +#endif + + return result; } int mmap_settle(void* address, size_t size, int fd, size_t offset) NOEXCEPT