diff --git a/include/bitcoin/database/impl/memory/mmap.ipp b/include/bitcoin/database/impl/memory/mmap.ipp index a3b8746c4..9e7f788cd 100644 --- a/include/bitcoin/database/impl/memory/mmap.ipp +++ b/include/bitcoin/database/impl/memory/mmap.ipp @@ -136,7 +136,7 @@ size_t CLASS::to_growth(size_t required) const NOEXCEPT return std::min(chunked, std::max(expanded, to_provision())); #else - // The classic mapping is file-backed, so growth is capacity. + // The native mapping is file-backed, so growth is capacity. return to_capacity(required); #endif } @@ -162,7 +162,7 @@ size_t CLASS::to_commitment() const NOEXCEPT const auto logical = logical_.load(); return std::min(to_provision(), std::max(logical, to_rows(to_chunk()))); #else - // The classic mapping is file-backed, so commitment is provisioning. + // The native mapping is file-backed, so commitment is provisioning. return to_provision(); #endif } diff --git a/include/bitcoin/database/impl/memory/mmap_dispatch.ipp b/include/bitcoin/database/impl/memory/mmap_dispatch.ipp index b71f3fe9c..94a202a4c 100644 --- a/include/bitcoin/database/impl/memory/mmap_dispatch.ipp +++ b/include/bitcoin/database/impl/memory/mmap_dispatch.ipp @@ -62,7 +62,6 @@ memory CLASS::get_filled(size_t offset, size_t size, const auto logical = to_width(logical_.load()); const auto capacity = to_width(capacity_.load()); const auto start = std::next(data, logical); - prepare(logical, capacity - logical); std::fill_n(start, capacity - logical, backfill); mark(logical, capacity - logical); } diff --git a/include/bitcoin/database/impl/memory/mmap_private.ipp b/include/bitcoin/database/impl/memory/mmap_private.ipp index 3fbe4fd24..d98f422d9 100644 --- a/include/bitcoin/database/impl/memory/mmap_private.ipp +++ b/include/bitcoin/database/impl/memory/mmap_private.ipp @@ -95,11 +95,7 @@ bool CLASS::unmap_all_(std::index_sequence) NOEXCEPT frontier_.store(zero); marks_.store(zero); dirty_.reset(); - intent_.reset(); - released_.reset(); - sweep_.reset(); words_ = zero; - engaged_.store(false); #endif return success; @@ -130,11 +126,7 @@ bool CLASS::remap_all_(size_t capacity, std::index_sequence) NOEXCEPT // Never results in unmapped. TEMPLATE template -bool CLASS::flush_(size_t - #if defined(MANAGE_STAGING) || defined(HAVE_MSC) - rows - #endif -) NOEXCEPT +bool CLASS::flush_(size_t rows) NOEXCEPT { #if defined(MANAGE_STAGING) // Transfer unflushed rows from anonymous memory to the file. Settled rows @@ -149,7 +141,7 @@ bool CLASS::flush_(size_t head_shared ? (::msync(memory_map_[Column], to, MS_SYNC) != fail) : transfer_(to)) && sync_(); -#elif defined(HAVE_MSC) +#else // unmap (and therefore msync) must be called before ftruncate. // "To flush all the dirty pages plus the metadata for the file and ensure // that they are physically written to disk..." @@ -157,16 +149,6 @@ bool CLASS::flush_(size_t const auto success = (::msync(memory_map_[Column], size, MS_SYNC) != fail) && (::fsync(opened_[Column]) != fail); -#else - // msync should not be required on modern linux, see linus et al. - // stackoverflow.com/questions/5902629/mmap-msync-and-linux-process-termination - // Linux: fsync "transfers ("flushes") all modified in-core data of - // (i.e., modified buffer cache pages for) the file referred to by the - // file descriptor fd to the disk device so all changed information - // can be retrieved even if the system crashes or is rebooted. This - // includes writing through or flushing a disk cache if present. The - // call blocks until the device reports that transfer has completed." - const auto success = ::fsync(opened_[Column]) != fail; #endif if (!success) @@ -197,7 +179,7 @@ bool CLASS::release_(size_t size) NOEXCEPT TEMPLATE template bool CLASS::unmap_(size_t - #if !defined(MANAGE_STAGING) + #if defined(HAVE_MSC) size #endif ) NOEXCEPT @@ -220,7 +202,7 @@ bool CLASS::unmap_(size_t memory_map_[Column] = {}; reserved_[Column] = zero; -#elif defined(HAVE_MSC) +#else // Windows cannot resize a mapped file. // msync requires the live mapping, ftruncate requires it gone. const auto synced = @@ -230,14 +212,6 @@ bool CLASS::unmap_(size_t const auto success = release_(size) && synced && (::ftruncate(opened_[Column], logical) != fail) && (::fsync(opened_[Column]) != fail); -#else - // POSIX permits resizing a mapped file. - const auto truncated = - (::ftruncate(opened_[Column], logical) != fail) - && (::fsync(opened_[Column]) != fail); - - // Order ensures release in case of truncate failure. - const auto success = release_(size) && truncated; #endif loaded_.store(false); @@ -258,7 +232,7 @@ bool CLASS::map_() NOEXCEPT return stage_(); #else // Cannot map empty file, and want minimum capacity, so expand as required. - // The classic mapping is file-backed, so commitment is provisioning. + // The native mapping is file-backed, so commitment is provisioning. // disk_full: space is set but no code is set with false return. const auto size = to_provision(); if (!resize_(size)) @@ -268,7 +242,7 @@ bool CLASS::map_() NOEXCEPT ::mmap(nullptr, to_width(size), PROT_READ | PROT_WRITE, MAP_SHARED, opened_[Column], 0)); - return finalize_(size); + return finalize_(); #endif } @@ -296,19 +270,13 @@ bool CLASS::remap_(size_t size) NOEXCEPT if (!resize_(size)) return false; -#if defined(HAVE_MSC) // mman-win32 mremap hack (umap/map) requires flags and file descriptor. memory_map_[Column] = system::pointer_cast( ::mremap_(memory_map_[Column], to_width(capacity_.load()), to_width(size), PROT_READ | PROT_WRITE, MAP_SHARED, opened_[Column])); -#else - memory_map_[Column] = system::pointer_cast( - ::mremap(memory_map_[Column], to_width(capacity_.load()), - to_width(size), MREMAP_MAYMOVE)); -#endif - return finalize_(size); + return finalize_(); #endif // MANAGE_STAGING } @@ -353,11 +321,7 @@ bool CLASS::resize_(size_t size) NOEXCEPT // Finalize failure results in unmapped. TEMPLATE template -bool CLASS::finalize_(size_t - #if !defined(HAVE_MSC) && !defined(WITHOUT_MADVISE) - size - #endif -) NOEXCEPT +bool CLASS::finalize_() NOEXCEPT { if (memory_map_[Column] == MAP_FAILED) { @@ -369,51 +333,6 @@ bool CLASS::finalize_(size_t return false; } -#if !defined(HAVE_MSC) && !defined(WITHOUT_MADVISE) - // Get page size (usually 4KB). - using namespace system; - const int page_size = ::sysconf(_SC_PAGESIZE); - const auto page = possible_narrow_sign_cast(page_size); - - // If not one bit then page size is not a power of two as required. - if (page_size == fail || !is_one(ones_count(page))) - { - set_first_code(error::sysconf_failure); - unmap_(size); - return false; - } - - // Align mapped bytes up to page boundary. - const auto max = sub1(page); - const auto target = to_width(size); - const auto align = bit_and(ceilinged_add(target, max), bit_not(max)); - - // Advice is elective (normal is the kernel default) and configured from - // the read pattern (see database::advice), as advising from the write - // pattern (structural) invites fault read amplification on random reads. - // Random access preloads (small heads, avoiding initial fault stalls). - if (access_ != advice::normal) - { - const auto random = (access_ != advice::sequential); - const auto preload = (access_ == advice::random); - const auto behavior = random ? MADV_RANDOM : MADV_SEQUENTIAL; - - for (size_t offset{}; offset < align; offset += advise_chunk) - { - const auto length = std::min(advise_chunk, align - offset); - const auto start = std::next(memory_map_[Column], offset); - - if (::madvise(start, length, behavior) == fail || (preload && - ::madvise(start, length, MADV_WILLNEED) == fail)) - { - set_first_code(error::madvise_failure); - unmap_(size); - return false; - } - } - } -#endif // !HAVE_MSC && !WITHOUT_MADVISE - loaded_.store(true); return true; } diff --git a/include/bitcoin/database/impl/memory/mmap_staging.ipp b/include/bitcoin/database/impl/memory/mmap_staging.ipp index b424f652c..c3d053f63 100644 --- a/include/bitcoin/database/impl/memory/mmap_staging.ipp +++ b/include/bitcoin/database/impl/memory/mmap_staging.ipp @@ -326,10 +326,6 @@ bool CLASS::stage_() NOEXCEPT const auto pages = ceilinged_divide(reserved, page_); words_ = ceilinged_divide(pages, page_bound); dirty_ = std::make_unique(words_); - intent_ = std::make_unique(words_); - released_ = std::make_unique(words_); - sweep_ = std::make_unique(words_); - writers_.store(zero); } } @@ -544,12 +540,6 @@ bool CLASS::commit_(size_t size) NOEXCEPT dirty_ = std::move(grown); words_ = words; - - // The replacement reservation is fully anonymous (content was - // copied above), so released and intent page state resets. - intent_ = std::make_unique(words); - released_ = std::make_unique(words); - sweep_ = std::make_unique(words); } } @@ -986,16 +976,9 @@ void CLASS::settler_run_() NOEXCEPT // before reading, so racing writes remark and transfer on the next pass // (torn disk pages are unreachable, as live heads are only trusted // following a clean close). -// -// Memory scarcity additionally pages the head to its own file (the windows -// model): transfer, then release cold clean pages to read-only file mappings -// (reclaimable cache), restored to anonymous by prepare() before any write. -// Release waits one pass after engagement so all writers declare intent. TEMPLATE void CLASS::head_run_() NOEXCEPT { - const auto scarce = system_memory() / evict_factor; - // Ticks without a mark before idle draining. auto mark = marks_.load(); auto transferred = mark; @@ -1012,7 +995,6 @@ void CLASS::head_run_() NOEXCEPT return; const auto top = marks_.load(); - const auto writes = top - mark; still = (top == mark) ? std::min(add1(still), idle_seconds) : zero; mark = top; @@ -1060,33 +1042,7 @@ void CLASS::head_run_() NOEXCEPT } } - // Available includes reclaimable file cache, which a loaded store - // keeps large while the kernel swaps cold anonymous pages, so free - // exhaustion also signals scarcity (anon is being displaced). - const auto scarcity = head_release && dirty_ && - ((system_available() < scarce) || (system_free() < scarce)); - - // Once engaged, a quiet instance converts independent of momentary - // scarcity: the signal clears as swap absorbs the hot set, but the - // swapped pages remain anonymous, so reads fault them back one at a - // time (a serial swap-in per probe). Conversion instead settles - // clean pages without read-back (the file holds their content), - // freeing swap and routing reads through the file mapping. - const auto engaged = engaged_.load(); - const auto quiet = writes < release_quiet; - const auto draining = engaged && quiet; - if (!scarcity && !draining && - ((still < idle_seconds) || (transferred == top))) - continue; - - // A write-hot instance neither transfers nor releases under - // scarcity: transferred pages re-dirty immediately (write - // amplification without release payoff, as hash-scattered writes - // into released pages each cost a segment restore, and the sweep - // otherwise re-releases restored segments every pass). Its - // anonymous set is left to swap (dirty-exempt) until quiescence, - // typically the phase change. - if (scarcity && !quiet) + if ((still < idle_seconds) || (transferred == top)) continue; { @@ -1095,15 +1051,8 @@ void CLASS::head_run_() NOEXCEPT if (!loaded_.load() || fault_.load()) continue; - if constexpr (head_release) - if (scarcity && !engaged) - engaged_.store(true); - - // Scarcity passes pace writeback for release: settle maps page - // cache content, and durability remains a clean close property, - // so sync applies only to idle draining. if (!transfer_(to_width(logical_.load())) || - (!scarcity && !sync_())) + !sync_()) { set_first_code(error::fsync_failure); continue; @@ -1112,11 +1061,6 @@ void CLASS::head_run_() NOEXCEPT // Discard the page cache copy of the transfer: the anonymous // head is the live copy, so caching the file doubles it. file_discard(opened_[zero]); - - // Quiet is assured here (hot scarcity skipped above, and idle - // draining implies sixty still seconds). - if (engaged && !release_pages_()) - continue; } transferred = top; @@ -1124,133 +1068,6 @@ void CLASS::head_run_() NOEXCEPT } } -// Release cold clean head page runs to read-only file mappings (reclaimable), -// full pages below logical only. Conversion is run-granular (release_chunk -// minimum) as each conversion splits a mapping: page granularity fragments -// the address space beyond what host memory management tolerates. Writer -// synchronization is a per-page bit protocol: prepare() declares intent then -// loads released; release stores released then loads intent (both -// sequentially consistent), so a run converts only when no write can land on -// it unrestored. A wrong release costs one restore. Conversion and restore -// serialize on restore_mutex_. -TEMPLATE -bool CLASS::release_pages_() NOEXCEPT -{ - using namespace system; - const auto bytes = to_width(logical_.load()); - const auto pages = bytes / page_; - const auto bound = std::min(words_, ceilinged_divide(pages, page_bound)); - const auto chunk = std::max(one, release_chunk / page_); - - std::unique_lock restore_lock(restore_mutex_); - - // Materialize candidacy (hot aging clears only the snapshot bits, so a - // concurrent declaration on a candidate page is retained for the live - // rechecks below). - for (size_t word{}; word < bound; ++word) - { - const auto hot = intent_[word].load(); - intent_[word].fetch_and(bit_not(hot)); - sweep_[word] = release_below(release_candidates( - dirty_[word].load(relaxed), hot, released_[word].load(relaxed)), - pages, word * page_bound); - } - - // Convert maximal candidate runs of at least chunk pages. - for (auto run = next_run(sweep_.get(), pages, zero); run.first < pages; - run = next_run(sweep_.get(), pages, run.second)) - { - if ((run.second - run.first) < chunk) - continue; - - const auto begin = run.first / page_bound; - const auto end = sub1(run.second) / page_bound; - const auto mask = [&](size_t word) NOEXCEPT - { - return page_mask(run.first, run.second, word * page_bound); - }; - - // Declare the release (prepare() restores from this point). - for (auto word = begin; word <= end; ++word) - released_[word].fetch_or(mask(word)); - - // An in-flight writer (counted, unaged) or raced intent or mark - // invalidates the conversion (whole run). The count loads first: a - // writer counted later observes released and restores, one drained - // earlier has published its marks (both sequentially consistent). - auto raced = is_nonzero(writers_.load()); - for (auto word = begin; (word <= end) && !raced; ++word) - raced = !is_zero(bit_and(mask(word), - bit_or(intent_[word].load(), dirty_[word].load()))); - - if (raced || (mmap_settle( - std::next(memory_map_[zero], run.first * page_), - (run.second - run.first) * page_, opened_[zero], - run.first * page_) == fail)) - { - for (auto word = begin; word <= end; ++word) - released_[word].fetch_and(bit_not(mask(word))); - - if (!raced) - { - set_first_code(error::mmap_failure); - return false; - } - } - } - - return true; -} - -// Restore released pages overlapping [offset, offset+size) to writable -// anonymous memory (content preserved by atomic installation), before a -// declared write. Restoration is segmented at release_chunk alignment: the -// containing segment restores whole (unifying any fragmentation within it) -// but never more, as released runs consolidate without bound and restoring -// a maximal run copies gigabytes per scattered write (a restore convoy). -TEMPLATE -void CLASS::restore_(size_t offset, size_t size) NOEXCEPT -{ - using namespace system; - std::unique_lock restore_lock(restore_mutex_); - - // Segments clamp to full pages below logical (as does release candidacy): - // the reservation above commitment is inaccessible (installation reads). - const auto pages = to_width(logical_.load()) / page_; - const auto span = std::max(one, release_chunk / page_); - const auto last = (offset + sub1(size)) / page_; - auto page = offset / page_; - page -= (page % span); - - for (; (page <= last) && (page < pages); page += span) - { - const auto stop = std::min(page + span, pages); - const auto mask = [&](size_t word) NOEXCEPT - { - return page_mask(page, stop, word * page_bound); - }; - - const auto begin = page / page_bound; - const auto end = sub1(stop) / page_bound; - auto any = false; - for (auto word = begin; (word <= end) && !any; ++word) - any = !is_zero(bit_and(released_[word].load(), mask(word))); - - if (!any) - continue; - - if (mmap_restore(std::next(memory_map_[zero], page * page_), - (stop - page) * page_) == fail) - { - set_first_code(error::mmap_failure); - return; - } - - for (auto word = begin; word <= end; ++word) - released_[word].fetch_and(bit_not(mask(word))); - } -} - // Settle up to chunk completed rows: write under the shared remap lock // (completed extents are immutable, writers proceed), convert under a brief // exclusive. Durability remains a snapshot property (no sync here). diff --git a/include/bitcoin/database/impl/memory/mmap_storage.ipp b/include/bitcoin/database/impl/memory/mmap_storage.ipp index df3095d4a..5976d7a57 100644 --- a/include/bitcoin/database/impl/memory/mmap_storage.ipp +++ b/include/bitcoin/database/impl/memory/mmap_storage.ipp @@ -187,43 +187,6 @@ code CLASS::reload() NOEXCEPT return error::reload_locked; } -TEMPLATE -void CLASS::prepare(size_t STAGING_ONLY(offset), - size_t STAGING_ONLY(size)) NOEXCEPT -{ -#if defined(MANAGE_STAGING) - if (is_zero(size) || !dirty_) - return; - - // Count the writer before loading released below (sequentially - // consistent), pairing with the release protocol: release either observes - // the count (and aborts) or this writer observes released (and restores). - // Intent bits age (hot sampling), so they cannot protect a write held - // in flight across passes; the count persists until mark. - writers_.fetch_add(one); - - if (!engaged_.load(relaxed)) - return; - - // Declare intent before the write (sequentially consistent, pairing with - // the release protocol), then restore any released page in the range. - auto restore = false; - auto page = offset / page_; - const auto end = (offset + sub1(size)) / page_; - while ((page <= end) && ((page / page_bound) < words_)) - { - const auto word = page / page_bound; - const auto flag = system::bit_right(page % page_bound); - intent_[word].fetch_or(flag); - restore |= !is_zero(system::bit_and(released_[word].load(), flag)); - ++page; - } - - if (restore) - restore_(offset, size); -#endif -} - TEMPLATE void CLASS::mark(size_t STAGING_ONLY(offset), size_t STAGING_ONLY(size)) NOEXCEPT @@ -235,12 +198,6 @@ void CLASS::mark(size_t STAGING_ONLY(offset), // Marks follow content writes; transfer clears before reading, so pages // remarked during a transfer are simply rewritten by the next pass. remark_(offset, size); - - // Uncount the writer after its marks (sequentially consistent), so a - // release pass loading a drained count observes the dirty bits. Only - // prepare() counts, so only mark() may uncount (transfer failure restores - // marks by remark_, as an unpaired uncount here corrupts the count). - writers_.fetch_sub(one); #endif } diff --git a/include/bitcoin/database/impl/primitives/arrayhead.ipp b/include/bitcoin/database/impl/primitives/arrayhead.ipp index 1748dd7dc..f0d32d1bb 100644 --- a/include/bitcoin/database/impl/primitives/arrayhead.ipp +++ b/include/bitcoin/database/impl/primitives/arrayhead.ipp @@ -71,7 +71,6 @@ bool CLASS::clear() NOEXCEPT // count to zero, which is picked up in arraymap::reset(). Body file size // remains unchanged and subject to initialization size at each startup. So // there is no reduction until restart, which can include config change. - file_.prepare(zero, size()); std::fill_n(ptr.data(), size(), system::bit_all); file_.mark(zero, size()); return set_body_count(zero); @@ -133,7 +132,6 @@ bool CLASS::set_body_count(const Link& count) NOEXCEPT // Body count is written as the first value in link size, but since // offsetting is a multiple of cell size, a full cell is consumed for it. // In case of nomap or disabled there are no cells, so file is link size. - file_.prepare(zero, Link::size); to_array(ptr.data()) = count; file_.mark(zero, Link::size); return true; @@ -188,7 +186,6 @@ bool CLASS::push(const Link& link, const Link& index) NOEXCEPT if (!ptr) return false; - file_.prepare(position, bucket_size); if constexpr (aligned) { // Writes full padded word (0x00 fill). diff --git a/include/bitcoin/database/impl/primitives/hashhead.ipp b/include/bitcoin/database/impl/primitives/hashhead.ipp index d546e399e..1a33fc092 100644 --- a/include/bitcoin/database/impl/primitives/hashhead.ipp +++ b/include/bitcoin/database/impl/primitives/hashhead.ipp @@ -72,7 +72,6 @@ bool CLASS::create() NOEXCEPT // std::memset/fill_n have identical performance (on win32). ////std::memset(ptr.data(), system::bit_all, allocation); - file_.prepare(start, allocation); std::fill_n(ptr.data(), allocation, system::bit_all); file_.mark(start, allocation); return set_body_count(zero); @@ -121,7 +120,6 @@ bool CLASS::set_body_count(const Link& count) NOEXCEPT // offsetting is a multiple of sell size, a full cell is consumed for it. // In case of disabled there are no cells, so file is link size. auto value = count.value; - file_.prepare(zero, Link::size); link_array(ptr.data()) = link_array(value); file_.mark(zero, Link::size); return true; @@ -216,7 +214,6 @@ inline bool CLASS::set_cell(bool& collision, bytes& next, const Link& current, if (is_null(raw)) return false; - file_.prepare(position, cell_size); const auto entropy = keys::thumb(key); if constexpr (aligned) { diff --git a/include/bitcoin/database/impl/primitives/nohead.ipp b/include/bitcoin/database/impl/primitives/nohead.ipp index 28670bee1..8d093e2ce 100644 --- a/include/bitcoin/database/impl/primitives/nohead.ipp +++ b/include/bitcoin/database/impl/primitives/nohead.ipp @@ -63,7 +63,6 @@ bool CLASS::clear() NOEXCEPT // count to zero, which is picked up in arraymap::reset(). Body file size // remains unchanged and subject to initialization size at each startup. So // there is no reduction until restart, which can include config change. - file_.prepare(zero, size()); std::fill_n(ptr.data(), size(), system::bit_all); file_.mark(zero, size()); return set_body_count(zero); @@ -125,7 +124,6 @@ bool CLASS::set_body_count(const Link& count) NOEXCEPT // Body count is written as the first value in link size, but since // offsetting is a multiple of cell size, a full cell is consumed for it. // In case of nomap or disabled there are no cells, so file is link size. - file_.prepare(zero, Link::size); to_array(ptr.data()) = count; file_.mark(zero, Link::size); return true; diff --git a/include/bitcoin/database/memory/interfaces/storage.hpp b/include/bitcoin/database/memory/interfaces/storage.hpp index 541fd81b4..f74d945aa 100644 --- a/include/bitcoin/database/memory/interfaces/storage.hpp +++ b/include/bitcoin/database/memory/interfaces/storage.hpp @@ -58,12 +58,6 @@ class storage /// Clear disk full condition, fails if fault, must be loaded, idempotent. virtual code reload() NOEXCEPT = 0; - /// Declare content mutation of size bytes at offset (rewritable tables), - /// before the write. Restores released pages; no effect where unneeded. - virtual void prepare(size_t, size_t) NOEXCEPT - { - } - /// Report content mutation of size bytes at offset (rewritable tables), /// after the write. Advisory dirty tracking; no effect where unneeded. virtual void mark(size_t, size_t) NOEXCEPT diff --git a/include/bitcoin/database/memory/mmap.hpp b/include/bitcoin/database/memory/mmap.hpp index d92bc2842..afd12a1aa 100644 --- a/include/bitcoin/database/memory/mmap.hpp +++ b/include/bitcoin/database/memory/mmap.hpp @@ -118,10 +118,6 @@ class mmap /// Clear disk full condition, fails if fault, must be loaded, idempotent. code reload() NOEXCEPT override; - /// Declare content mutation, restoring released pages (unstaged - /// instances under the staging backend only; no effect otherwise). - void prepare(size_t offset, size_t size) NOEXCEPT override; - /// Report content mutation (advisory page-dirty tracking, unstaged /// instances under the staging backend only; no effect otherwise). void mark(size_t offset, size_t size) NOEXCEPT override; @@ -218,7 +214,6 @@ class mmap static constexpr size_t chunk_scale = 256; static constexpr size_t evict_chunk = system::power2(30u); static constexpr size_t compress_factor = 32; - static constexpr size_t evict_factor = 32; // Body eviction leads kernel reclaim: sweeping at the reclaim watermark // concedes the choice of victim, and the kernel takes anonymous heads @@ -233,20 +228,13 @@ class mmap static constexpr size_t touch_seconds = 4; static constexpr size_t touch_span = 16384; - // Release conversion granularity: chunked runs bound address space - // fragmentation (each conversion splits a mapping) to the measured flat - // zone of host memory management (heads / chunk fragments worst case). - static constexpr size_t release_chunk = system::power2(20u); - static constexpr size_t release_quiet = 128; - static constexpr bool head_release = false; - // Map heads writable-shared from their files (the native windows model) // instead of anonymously with a dirty-page writer. Head reclaim is then // kernel writeback of a bounded rewrite-in-place mapping (clean pages // drop) rather than swap, which anonymous pages alone require. Bodies // remain staged, so the unbounded append writeback that motivates dirty - // ratio tuning does not return with it. Excludes head_release (nothing - // to release) and the dirty bitmap (nothing to transfer). + // ratio tuning does not return with it. Excludes the dirty bitmap + // (nothing to transfer). static constexpr bool head_shared = false; static constexpr size_t headroom = 4; #if defined(STAGING_TELEMETRY) @@ -281,7 +269,7 @@ class mmap template bool resize_(size_t size) NOEXCEPT; template - bool finalize_(size_t size) NOEXCEPT; + bool finalize_() NOEXCEPT; #if defined(MANAGE_STAGING) // staging dispatch, not thread safe. @@ -321,11 +309,6 @@ class mmap bool sync_() NOEXCEPT; void remark_(size_t offset, size_t size) NOEXCEPT; - // head page release (unstaged instances), synchronized with writers by - // the prepare/release bit protocol (see release_pages_). - bool release_pages_() NOEXCEPT; - void restore_(size_t offset, size_t size) NOEXCEPT; - // settle scheduler (instance-owned thread, load/unload lifecycle). void settler_start_() NOEXCEPT; void settler_stop_() NOEXCEPT; @@ -398,24 +381,8 @@ class mmap // These are protected by remap_mutex_. std::unique_ptr dirty_{}; - std::unique_ptr intent_{}; - std::unique_ptr released_{}; size_t words_{}; - // Sweep scratch (candidate/released word snapshots), protected by - // restore_mutex_. - std::unique_ptr sweep_{}; - - // Set when the first head page releases (gates the prepare fast path). - std::atomic_bool engaged_{}; - - // Writers between prepare and mark (unaged, unlike intent bits), so a - // release pass cannot settle under a preempted in-flight write. - std::atomic writers_{}; - - // Serializes page release against restore (prepare slow path). - mutable std::mutex restore_mutex_{}; - // Serializes transfer passes (settler tick against flush), as concurrent // passes split the claimed dirty set, allowing a flush to complete while // claimed pages remain unwritten (a stale snapshot copy). diff --git a/include/bitcoin/database/memory/mstage.hpp b/include/bitcoin/database/memory/mstage.hpp index 43a8b6f2c..2fc3f5251 100644 --- a/include/bitcoin/database/memory/mstage.hpp +++ b/include/bitcoin/database/memory/mstage.hpp @@ -21,15 +21,12 @@ #include -#if defined(HAVE_APPLE) +// The native windows mapped-file behavior is the model staging emulates, +// so staging is the build for all other platforms. +#if !defined(HAVE_MSC) #define MANAGE_STAGING #endif -// The native windows mapped-file behavior is the model staging emulates. -#if defined(MANAGE_STAGING) && defined(HAVE_MSC) - #error "MANAGE_STAGING is not supported on Windows." -#endif - #if defined(MANAGE_STAGING) #define STAGING_ONLY(name) name #else @@ -69,10 +66,6 @@ int mmap_resident(const void* address, size_t size, /// Discard unmapped page cache of a file (mapped pages are unaffected). int file_discard(int fd) NOEXCEPT; -/// Atomically replace a released (read-only file-backed) range with writable -/// anonymous memory carrying its content (readers never observe zeros). -int mmap_restore(void* address, size_t size) NOEXCEPT; - /// Full-transfer positional file read/write (false on failure or early eof). bool pread_all(int fd, uint8_t* to, size_t size, size_t offset) NOEXCEPT; bool pwrite_all(int fd, const uint8_t* from, size_t size, diff --git a/include/bitcoin/database/memory/utilities.hpp b/include/bitcoin/database/memory/utilities.hpp index 8f273f26d..b1d88cf86 100644 --- a/include/bitcoin/database/memory/utilities.hpp +++ b/include/bitcoin/database/memory/utilities.hpp @@ -51,89 +51,6 @@ BCD_API size_t system_pressure() NOEXCEPT; /// if failed or the platform provides no source. BCD_API uint64_t system_compressed() NOEXCEPT; -/// Head-release page-run helpers (pure, unit tested). Pages are tracked as -/// bits in 64-bit words (low order bit is the lowest page of the word). A -/// release candidate page is clean (not dirty), cold (not recently written) -/// and not already released; candidacy is limited to full pages below the -/// logical page count. -/// --------------------------------------------------------------------------- - -/// Candidate bits given page-state words. -constexpr uint64_t release_candidates(uint64_t dirty, uint64_t hot, - uint64_t released) NOEXCEPT -{ - using namespace system; - return bit_not(bit_or(dirty, bit_or(hot, released))); -} - -/// Retain candidacy below the page count, for the word starting at page -/// 'first' (a full word of candidacy above the count clears to zero). -constexpr uint64_t release_below(uint64_t candidates, size_t pages, - size_t first) NOEXCEPT -{ - using namespace system; - return (pages <= first) ? zero : - (pages - first >= to_bits(sizeof(uint64_t))) ? candidates : - bit_and(candidates, unmask_right(pages - first)); -} - -/// Bits for pages [lo, hi) clamped to the word of 64 pages starting at page -/// 'first', empty if the ranges do not intersect. -constexpr uint64_t page_mask(size_t lo, size_t hi, size_t first) NOEXCEPT -{ - using namespace system; - constexpr auto bits = to_bits(sizeof(uint64_t)); - const auto begin = std::max(lo, first); - const auto end = std::min(hi, first + bits); - return (begin >= end) ? zero : bit_and( - (end - first >= bits) ? bit_all : - unmask_right(end - first), - mask_right(begin - first)); -} - -/// The maximal run of set bits at or above 'page' within 'pages', as the -/// right-open page range [first, second), empty (equal) if none. -inline std::pair next_run(const uint64_t* words, size_t pages, - size_t page) NOEXCEPT -{ - using namespace system; - constexpr auto bits = to_bits(sizeof(uint64_t)); - - // Find first set bit at or above page. - while ((page < pages) && !get_right(words[page / bits], page % bits)) - ++page; - - // Find first clear bit above start. - auto end = page; - while ((end < pages) && get_right(words[end / bits], end % bits)) - ++end; - - return { page, end }; -} - -/// The maximal run of set bits containing 'page' within 'pages', as the -/// right-open page range [first, second), empty (equal) if page is not set. -inline std::pair bit_run(const uint64_t* words, size_t pages, - size_t page) NOEXCEPT -{ - using namespace system; - constexpr auto bits = to_bits(sizeof(uint64_t)); - - if ((page >= pages) || !get_right(words[page / bits], page % bits)) - return { page, page }; - - auto start = page; - while (!is_zero(start) && - get_right(words[sub1(start) / bits], sub1(start) % bits)) - --start; - - auto end = add1(page); - while ((end < pages) && get_right(words[end / bits], end % bits)) - ++end; - - return { start, end }; -} - /// C++26: std::atomic::fetch_max template = true> Integral fetch_max(std::atomic& atomic, Integral value) NOEXCEPT diff --git a/src/memory/mstage.cpp b/src/memory/mstage.cpp index 4235ac89b..1a1244b18 100644 --- a/src/memory/mstage.cpp +++ b/src/memory/mstage.cpp @@ -28,10 +28,6 @@ #include #include #include -#if defined(HAVE_APPLE) - #include - #include -#endif #if defined(HAVE_LINUX) #include #endif @@ -142,63 +138,27 @@ int mmap_unsettle(void* address, size_t size) NOEXCEPT int mmap_evict(void* address, size_t size) NOEXCEPT { - // Sync first: a settle write may not yet be written back, and bare - // invalidation of a dirty page discards it (clean pages sync free). -#if defined(MADV_PAGEOUT) + // The apple sdk defines MADV_PAGEOUT but xnu refuses it (ENOTSUP), so + // the gate must be platform-true, not macro-present: darwin retains + // invalidation (MS_INVALIDATE drops clean ubc pages on mac). +#if defined(MADV_PAGEOUT) && !defined(HAVE_APPLE) // MS_INVALIDATE does not drop clean page cache on linux (it invalidates // other mappings of the file), leaving the eviction sweep inert: body // cache pressure stands and the kernel preserves it by swapping the // anonymous heads. Pageout reclaims the mapped range (dirty pages write - // back, clean pages drop, later reads fault back from the file). - if (::msync(address, size, MS_SYNC) == -1) - return -1; - + // back, clean pages drop, later reads fault back from the file). No + // sync first: pageout never discards dirty content (unlike bare + // invalidation), and a synchronous flush here serializes the settle + // write path behind a device flush barrier on every sweep (durability + // remains the clean close). return ::madvise(address, size, MADV_PAGEOUT); #else + // Sync first: a settle write may not yet be written back, and bare + // invalidation of a dirty page discards it (clean pages sync free). return ::msync(address, size, MS_SYNC | MS_INVALIDATE); #endif } -// Atomically install the source anonymous mapping over the target range, -// consuming the source (readers see old or new bytes, never zeros). -static int mmap_install(void* address, void* source, size_t size) NOEXCEPT -{ -#if defined(HAVE_APPLE) - auto target = reinterpret_cast(address); - vm_prot_t current{}; - vm_prot_t maximum{}; - if (::mach_vm_remap(::mach_task_self(), &target, size, 0, - VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE, ::mach_task_self(), - reinterpret_cast(source), TRUE, ¤t, - &maximum, VM_INHERIT_DEFAULT) != KERN_SUCCESS) - return -1; - - return ::munmap(source, size); -#else - return ::mremap(source, size, size, MREMAP_MAYMOVE | MREMAP_FIXED, - address) == MAP_FAILED ? -1 : 0; -#endif -} - -int mmap_restore(void* address, size_t size) NOEXCEPT -{ - // Anonymous staging copy of the current (file) content. - const auto source = ::mmap(nullptr, size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - - if (source == MAP_FAILED) - return -1; - - std::memcpy(source, address, size); - if (mmap_install(address, source, size) == -1) - { - ::munmap(source, size); - return -1; - } - - return 0; -} - bool pread_all(int fd, uint8_t* to, size_t size, size_t offset) NOEXCEPT { while (!is_zero(size)) diff --git a/test/memory/utilities.cpp b/test/memory/utilities.cpp index f532a1863..9d68b9955 100644 --- a/test/memory/utilities.cpp +++ b/test/memory/utilities.cpp @@ -35,155 +35,5 @@ BOOST_AUTO_TEST_CASE(memory_utilities__system_memory__always__nonzero) BOOST_REQUIRE(is_nonzero(system_memory())); } -// release_candidates - -BOOST_AUTO_TEST_CASE(memory_utilities__release_candidates__none__all) -{ - BOOST_REQUIRE_EQUAL(release_candidates(0, 0, 0), max_uint64); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__release_candidates__each__excluded) -{ - BOOST_REQUIRE_EQUAL(release_candidates(0b0001, 0b0010, 0b0100), - system::bit_not(0b0111)); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__release_candidates__all__none) -{ - BOOST_REQUIRE_EQUAL(release_candidates(max_uint64, 0, 0), zero); - BOOST_REQUIRE_EQUAL(release_candidates(0, max_uint64, 0), zero); - BOOST_REQUIRE_EQUAL(release_candidates(0, 0, max_uint64), zero); -} - -// release_below -// The boundary word masking defect class: candidacy must retain LOW bits -// (pages below the count), never the high complement. - -BOOST_AUTO_TEST_CASE(memory_utilities__release_below__word_above_count__zero) -{ - BOOST_REQUIRE_EQUAL(release_below(max_uint64, 64, 64), zero); - BOOST_REQUIRE_EQUAL(release_below(max_uint64, 10, 64), zero); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__release_below__word_below_count__all) -{ - BOOST_REQUIRE_EQUAL(release_below(max_uint64, 64, 0), max_uint64); - BOOST_REQUIRE_EQUAL(release_below(max_uint64, 128, 64), max_uint64); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__release_below__boundary__low_bits) -{ - // One full page: only bit zero (the historical inversion released 1..63). - BOOST_REQUIRE_EQUAL(release_below(max_uint64, 1, 0), 0b0001u); - BOOST_REQUIRE_EQUAL(release_below(max_uint64, 3, 0), 0b0111u); - - // Sixty-five pages: boundary word (first=64) retains only bit zero. - BOOST_REQUIRE_EQUAL(release_below(max_uint64, 65, 64), 0b0001u); -} - -// page_mask - -BOOST_AUTO_TEST_CASE(memory_utilities__page_mask__disjoint__zero) -{ - BOOST_REQUIRE_EQUAL(page_mask(0, 10, 64), zero); - BOOST_REQUIRE_EQUAL(page_mask(128, 130, 64), zero); - BOOST_REQUIRE_EQUAL(page_mask(10, 10, 0), zero); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__page_mask__contained__expected) -{ - BOOST_REQUIRE_EQUAL(page_mask(0, 1, 0), 0b0001u); - BOOST_REQUIRE_EQUAL(page_mask(1, 3, 0), 0b0110u); - BOOST_REQUIRE_EQUAL(page_mask(65, 67, 64), 0b0110u); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__page_mask__spanning__clamped) -{ - // Run [60, 70) in word zero: bits 60..63; in word one: bits 0..5. - BOOST_REQUIRE_EQUAL(page_mask(60, 70, 0), - system::bit_and(max_uint64, system::mask_right(60))); - BOOST_REQUIRE_EQUAL(page_mask(60, 70, 64), system::unmask_right(6)); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__page_mask__full_word__all) -{ - BOOST_REQUIRE_EQUAL(page_mask(0, 128, 64), max_uint64); -} - -// next_run - -BOOST_AUTO_TEST_CASE(memory_utilities__next_run__empty__none) -{ - const uint64_t words[]{ 0, 0 }; - const auto run = next_run(words, 128, 0); - BOOST_REQUIRE_EQUAL(run.first, run.second); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__next_run__single_page__found) -{ - const uint64_t words[]{ 0b01000, 0 }; - const auto run = next_run(words, 128, 0); - BOOST_REQUIRE_EQUAL(run.first, 3u); - BOOST_REQUIRE_EQUAL(run.second, 4u); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__next_run__word_spanning__joined) -{ - // Bits 62..63 of word zero and 0..2 of word one: run [62, 67). - const uint64_t words[]{ system::mask_right(62), 0b0111 }; - const auto run = next_run(words, 128, 0); - BOOST_REQUIRE_EQUAL(run.first, 62u); - BOOST_REQUIRE_EQUAL(run.second, 67u); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__next_run__from_skips__second) -{ - const uint64_t words[]{ 0b0011, 0b0011 }; - const auto run = next_run(words, 128, 2); - BOOST_REQUIRE_EQUAL(run.first, 64u); - BOOST_REQUIRE_EQUAL(run.second, 66u); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__next_run__page_bound__clipped) -{ - // All set but only 70 pages exist: run [0, 70). - const uint64_t words[]{ max_uint64, max_uint64 }; - const auto run = next_run(words, 70, 0); - BOOST_REQUIRE_EQUAL(run.first, zero); - BOOST_REQUIRE_EQUAL(run.second, 70u); -} - -// bit_run - -BOOST_AUTO_TEST_CASE(memory_utilities__bit_run__unset_page__empty) -{ - const uint64_t words[]{ 0b0110, 0 }; - const auto run = bit_run(words, 128, 3); - BOOST_REQUIRE_EQUAL(run.first, run.second); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__bit_run__interior_page__extent) -{ - const uint64_t words[]{ 0b111100, 0 }; - const auto run = bit_run(words, 128, 3); - BOOST_REQUIRE_EQUAL(run.first, 2u); - BOOST_REQUIRE_EQUAL(run.second, 6u); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__bit_run__word_spanning__extent) -{ - // Bits 63 of word zero and 0..1 of word one: run [63, 66) from page 64. - const uint64_t words[]{ system::bit_left(0), 0b0011 }; - const auto run = bit_run(words, 128, 64); - BOOST_REQUIRE_EQUAL(run.first, 63u); - BOOST_REQUIRE_EQUAL(run.second, 66u); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__bit_run__beyond_pages__empty) -{ - const uint64_t words[]{ max_uint64, max_uint64 }; - const auto run = bit_run(words, 64, 100); - BOOST_REQUIRE_EQUAL(run.first, run.second); -} BOOST_AUTO_TEST_SUITE_END()