Maximize mmap disk utilization in full condition, macOS mmap WIP.

builds/gnu/NEWS

@@ -1 +1 @@
-See https://libbitcoin.org
+See https://libbitcoin.info

include/bitcoin/database/impl/memory/mmap.ipp

@@ -39,6 +39,7 @@ CLASS::mmap(const path& filename, const storage_settings& settings,
   : filenames_{ filename },
     minimum_(to_rows(settings.size)),
     expansion_(settings.rate),
+    headroom_(system::possible_narrow_cast<size_t>(settings.headroom)),
     access_(settings.access),
     random_(random),
     staged_(staged),
@@ -53,6 +54,7 @@ CLASS::mmap(const paths& filenames, const storage_settings& settings,
   : filenames_(filenames),
     minimum_(to_rows(settings.size)),
     expansion_(settings.rate),
+    headroom_(system::possible_narrow_cast<size_t>(settings.headroom)),
     access_(settings.access),
     random_(random),
     staged_(staged),

include/bitcoin/database/impl/memory/mmap_private.ipp

@@ -107,11 +107,32 @@ bool CLASS::unmap_all_(std::index_sequence<Index...>) NOEXCEPT
 
 TEMPLATE
 template <size_t... Index>
-bool CLASS::remap_all_(size_t capacity, std::index_sequence<Index...>) NOEXCEPT
+bool CLASS::remap_all_(size_t capacity, std::index_sequence<Index...>,
+    bool final) NOEXCEPT
 {
-    if (!(remap_<Index>(capacity) && ...))
+    // Probe the wave's disk requirement before touching any column file: a
+    // refused wave then retains no surplus provisioning (columns cannot be
+    // trimmed after a partial wave, as msc cannot shrink a mapped file).
+    if (!probe_(capacity))
     {
-        capacity_.store(zero);
+        if (final)
+        {
+            using namespace system;
+            set_disk_space(ceilinged_add(headroom_, ceilinged_multiply(
+                floored_subtract(capacity, file_.load()), stride)));
+        }
+
+        return false;
+    }
+
+    if (!(remap_<Index>(capacity, final) && ...))
+    {
+        // A non-final refusal leaves the maps and capacity intact for the
+        // caller's reduced retry (columns already grown by the refused
+        // attempt harmlessly retain surplus commitment or provisioning).
+        if (final)
+            capacity_.store(zero);
+
         return false;
     }
 
@@ -276,7 +297,7 @@ bool CLASS::map_() NOEXCEPT
 // Remapping has no effect on logical size, sets map_/capacity_.
 TEMPLATE
 template <size_t Column>
-bool CLASS::remap_(size_t size) NOEXCEPT
+bool CLASS::remap_(size_t size, bool final) NOEXCEPT
 {
     BC_ASSERT(size >= logical_.load());
 
@@ -288,12 +309,12 @@ bool CLASS::remap_(size_t size) NOEXCEPT
     // The file is preallocated to capacity, preserving disk full detection at
     // allocation, and growth commits reserved anonymous pages in place, so no
     // mapping is released and the map base is stable within the reservation.
-    if (!resize_<Column>(size))
+    if (!resize_<Column>(size, final))
         return false;
 
-    return commit_<Column>(size);
+    return commit_<Column>(size, final);
 #else
-    if (!resize_<Column>(size))
+    if (!resize_<Column>(size, final))
         return false;
 
 #if defined(HAVE_MSC)
@@ -315,30 +336,35 @@ bool CLASS::remap_(size_t size) NOEXCEPT
 // disk_full: space is set but no code is set with false return.
 TEMPLATE
 template <size_t Column>
-bool CLASS::resize_(size_t size) NOEXCEPT
+bool CLASS::resize_(size_t size, bool final) NOEXCEPT
 {
     // The file is provisioned ahead of commitment, so growth within the
     // provisioned extent requires no disk operation (the space is reserved).
     const auto extent = file_.load();
     if (size <= extent)
         return true;
 
+    using namespace system;
     const auto target = to_width<Column>(size);
     const auto capacity = to_width<Column>(extent);
 
-    // Disk full detection, any other failure is an abort.
+    // Disk full detection, any other failure is an abort. The wave probe
+    // (remap_all_) precedes, so refusal here is a raced foreign consumer.
 #if !defined(WITHOUT_FALLOCATE)
     if (::fallocate(opened_[Column], 0, capacity, target - capacity) == fail)
 #else
     if (::ftruncate(opened_[Column], target) == fail)
 #endif
     {
         // Disk full is the only restartable store failure (leave mapped).
+        // A non-final refusal is not published: the caller retries reduced.
+        // The published requirement includes the headroom (a retry probes).
         if (errno == ENOSPC)
         {
-            using namespace system;
-            set_disk_space(ceilinged_multiply(floored_subtract(size, extent),
-                stride));
+            if (final)
+                set_disk_space(ceilinged_add(headroom_, ceilinged_multiply(
+                    floored_subtract(size, extent), stride)));
+
             return false;
         }
 
@@ -350,6 +376,42 @@ bool CLASS::resize_(size_t size) NOEXCEPT
     return true;
 }
 
+// The wave probe reserves the whole extension plus headroom on the store
+// volume (column widths sum to the stride), released upon the grant: a
+// refused wave touches no column file, and a granted one leaves the
+// headroom unclaimed.
+TEMPLATE
+bool CLASS::probe_(size_t capacity) NOEXCEPT
+{
+    using namespace system;
+    const auto bytes = ceilinged_multiply(
+        floored_subtract(capacity, file_.load()), stride);
+
+    if (is_zero(bytes))
+        return true;
+
+    auto name = filenames_.front();
+    name += ".probe";
+    auto probe = file::invalid;
+    if (file::create_file(name))
+        probe = file::open(name);
+
+    const auto reserve = ceilinged_add(bytes, headroom_);
+#if !defined(WITHOUT_FALLOCATE)
+    const auto held = (probe != file::invalid) &&
+        (::fallocate(probe, 0, zero, reserve) != fail);
+#else
+    const auto held = (probe != file::invalid) &&
+        (::ftruncate(probe, reserve) != fail);
+#endif
+
+    if (probe != file::invalid)
+        file::close(probe);
+
+    file::remove(name);
+    return held;
+}
+
 // Finalize failure results in unmapped.
 TEMPLATE
 template <size_t Column>

include/bitcoin/database/impl/memory/mmap_staging.ipp

@@ -160,35 +160,42 @@ size_t CLASS::frontier() const NOEXCEPT
 
 #if defined(MANAGE_STAGING)
 
+// Claim and record an extent under one lock: a claim never exists outside
+// the ring and the ring is start-ordered, so the frontier can never pass an
+// unwritten extent (claim-then-record raced the frontier past the claim).
+// Returns eof (unclaimed) on insufficient capacity, fault, or disk full.
 TEMPLATE
-void CLASS::record_(size_t start, size_t count) NOEXCEPT
+size_t CLASS::record_(size_t count) NOEXCEPT
 {
-    if (!staged_ || is_zero(count))
-        return;
-
     std::unique_lock extent_lock(extent_mutex_);
 
+    if (is_zero(count))
+        return logical_.load();
+
     maintain_();
 
     using namespace system;
     auto [head, size] = unpack_word<uint64_t>(window_.load(relaxed));
 
     // A full ring waits on completions (extents are allocation-coarse, so
-    // saturation implies extreme concurrency). An unrecorded extent would be
-    // unsafe: an emptied ring advances the frontier to logical, so untracked
-    // incomplete writes could settle. Completions are lock-free, so waiting
-    // needs only this thread's own maintenance; fault or disk full releases
-    // the wait (recording is then moot, as recovery discards the ring).
+    // saturation implies extreme concurrency). Completions are lock-free, so
+    // waiting needs only this thread's own maintenance; fault or disk full
+    // releases the wait unclaimed (the write then fails fast).
     while (size == extents)
     {
         if (fault_.load() || !is_zero(space_.load()))
-            return;
+            return storage::eof;
 
         std::this_thread::yield();
         maintain_();
         std::tie(head, size) = unpack_word<uint64_t>(window_.load(relaxed));
     }
 
+    const auto start = logical_.load();
+    if (is_add_overflow(start, count) ||
+        ((start + count) > capacity_.load()))
+        return storage::eof;
+
     auto& record = ring_.at((head + size) % extents);
     const auto generation = bit_and<uint64_t>(add1(shift_right<uint64_t>(
         record.state.load(relaxed), generation_shift)), generation_mask);
@@ -206,6 +213,10 @@ void CLASS::record_(size_t start, size_t count) NOEXCEPT
     window_.store(pack_word<uint64_t>(head, add1(size)), release);
     if (is_zero(size))
         frontier_.store(start);
+
+    logical_.store(start + count);
+    check_invariants_();
+    return start;
 }
 
 // Pop completed extents from the head, advancing the frontier (locked).
@@ -397,7 +408,7 @@ bool CLASS::stage_() NOEXCEPT
     const auto settled = page_floor(to_width<Column>(settled_.load()));
 
     if ((target > settled) && (mmap_commit(std::next(memory_map_[Column],
-        settled), target - settled) == fail))
+        settled), target - settled, headroom_) == fail))
     {
         teardown_<Column>(error::mmap_failure);
         return false;
@@ -440,13 +451,17 @@ bool CLASS::stage_() NOEXCEPT
     return true;
 }
 
-// Commit failure results in unmapped.
+// Commit failure results in unmapped when final (the default); a non-final
+// refusal (in-reservation commit, replacement reservation, or replacement
+// commit) returns false with the standing mapping untouched, so the caller
+// may iterate a reduced request (admission is evaluated per request, so a
+// refused amortization step does not imply exhaustion).
 // Growth within the reservation commits pages in place (stable map base); an
 // exhausted reservation is replaced and its unsettled content copied, under
 // the exclusive remap lock held by the caller.
 TEMPLATE
 template <size_t Column>
-bool CLASS::commit_(size_t size) NOEXCEPT
+bool CLASS::commit_(size_t size, bool final) NOEXCEPT
 {
     const auto target = to_width<Column>(size);
 
@@ -476,9 +491,11 @@ bool CLASS::commit_(size_t size) NOEXCEPT
         const auto from = std::max(settled, current);
 
         if ((target > from) && (mmap_commit(std::next(memory_map_[Column], from),
-            target - from) == fail))
+            target - from, headroom_) == fail))
         {
-            teardown_<Column>(error::mmap_failure);
+            if (final)
+                teardown_<Column>(error::mmap_failure);
+
             return false;
         }
 
@@ -496,7 +513,9 @@ bool CLASS::commit_(size_t size) NOEXCEPT
 
     if (replace == MAP_FAILED)
     {
-        teardown_<Column>(error::mmap_failure);
+        if (final)
+            teardown_<Column>(error::mmap_failure);
+
         return false;
     }
 
@@ -530,10 +549,19 @@ bool CLASS::commit_(size_t size) NOEXCEPT
         return true;
     }
 
-    if (mmap_commit(std::next(base, settled), target - settled) == fail)
+    // The replacement commit spans the unsettled prefix, transiently charged
+    // over the standing reservation, so its refusal is the largest single
+    // admission of the design: non-final refusal leaves the standing mapping
+    // untouched for the caller's reduced retry (a reduced ask fits within
+    // the standing reservation), and settle drainage shrinks the span, so a
+    // necessity refusal pauses recoverable rather than tearing down.
+    if (mmap_commit(std::next(base, settled), target - settled,
+        headroom_) == fail)
     {
         ::munmap(replace, reserved);
-        teardown_<Column>(error::mmap_failure);
+        if (final)
+            teardown_<Column>(error::mmap_failure);
+
         return false;
     }