Skip to content

[#290] Report leftover MVV mark bits in IntegrityCheck #291

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vharseko merged 2 commits into from
Jul 22, 2026
Merged

[#290] Report leftover MVV mark bits in IntegrityCheck #291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e9efe41
Report leftover MVV mark bits in IntegrityCheck
vharseko Jul 21, 2026
a69c0b8
Compare fault counts in the icheck -P gate, not the capped fault list
vharseko Jul 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 51 additions & 7 deletions persistit/core/src/main/java/com/persistit/IntegrityCheck.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ public class IntegrityCheck extends Task {

private Volume _currentVolume;
private Tree _currentTree;
private long _currentPage;
private LongBitSet _usedPageBits = new LongBitSet();
private long _totalPages = 0;
private long _pagesVisited = 0;
Expand All @@ -79,6 +80,8 @@ public class IntegrityCheck extends Task {
private boolean _csv;

private final ArrayList<Fault> _faults = new ArrayList<Fault>();
private int _faultCount;
private int _markedVersionFaultCount;
private final ArrayList<CleanupIndexHole> _holes = new ArrayList<CleanupIndexHole>();

// Used in checking long values
Expand All @@ -98,6 +101,7 @@ private static class Counters {
private long _mvvCount = 0;
private long _mvvOverhead = 0;
private long _mvvAntiValues = 0;
private long _markedVersionCount = 0;
private long _pruningErrorCount = 0;
private long _prunedPageCount = 0;
private long _garbagePageCount = 0;
Expand All @@ -118,6 +122,7 @@ private static class Counters {
_mvvCount = counters._mvvCount;
_mvvOverhead = counters._mvvOverhead;
_mvvAntiValues = counters._mvvAntiValues;
_markedVersionCount = counters._markedVersionCount;
_pruningErrorCount = counters._pruningErrorCount;
_prunedPageCount = counters._prunedPageCount;
_garbagePageCount = counters._garbagePageCount;
Expand All @@ -135,6 +140,7 @@ void difference(final Counters counters) {
_mvvCount = counters._mvvCount - _mvvCount;
_mvvOverhead = counters._mvvOverhead - _mvvOverhead;
_mvvAntiValues = counters._mvvAntiValues - _mvvAntiValues;
_markedVersionCount = counters._markedVersionCount - _markedVersionCount;
_pruningErrorCount = counters._pruningErrorCount - _pruningErrorCount;
_prunedPageCount = counters._prunedPageCount - _prunedPageCount;
_garbagePageCount = counters._garbagePageCount - _garbagePageCount;
Expand All @@ -144,20 +150,21 @@ void difference(final Counters counters) {
public String toString() {
return String.format("Index pages/bytes: %,d / %,d Data pages/bytes: %,d / %,d"
+ " LongRec pages/bytes: %,d / %,d MVV pages/records/bytes/antivalues: "
+ "%,d / %,d / %,d / %,d Holes %,d Pages pruned %,d", _indexPageCount, _indexBytesInUse,
_dataPageCount, _dataBytesInUse, _longRecordPageCount, _longRecordBytesInUse, _mvvPageCount,
_mvvCount, _mvvOverhead, _mvvAntiValues, _indexHoleCount, _prunedPageCount);
+ "%,d / %,d / %,d / %,d Holes %,d Pages pruned %,d Marked versions %,d", _indexPageCount,
_indexBytesInUse, _dataPageCount, _dataBytesInUse, _longRecordPageCount, _longRecordBytesInUse,
_mvvPageCount, _mvvCount, _mvvOverhead, _mvvAntiValues, _indexHoleCount, _prunedPageCount,
_markedVersionCount);
}

private String toCSV() {
return String.format("%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d", _indexPageCount, _indexBytesInUse,
return String.format("%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d", _indexPageCount, _indexBytesInUse,
_dataPageCount, _dataBytesInUse, _longRecordPageCount, _longRecordBytesInUse, _mvvPageCount,
_mvvCount, _mvvOverhead, _mvvAntiValues, _indexHoleCount, _prunedPageCount);
_mvvCount, _mvvOverhead, _mvvAntiValues, _indexHoleCount, _prunedPageCount, _markedVersionCount);
}

private final static String CSV_HEADERS = "IndexPages,IndexBytes,"
+ "DataPages,DataBytes,LongRecordPages,LongRecordBytes,MvvPages,"
+ "MvvRecords,MvvOverhead,MvvAntiValues,IndexHoles,PrunedPages";
+ "MvvRecords,MvvOverhead,MvvAntiValues,IndexHoles,PrunedPages,MarkedVersions";

}

Expand Down Expand Up @@ -186,6 +193,19 @@ public void sawVersion(final long version, final int offset, final int valueLeng
protected void visitDataRecord(final Key key, final int foundAt, final int tail, final int klength,
final int offset, final int length, final byte[] bytes) throws PersistitException {
MVV.visitAllVersions(_versionVisitor, bytes, offset, length);
/*
* Scanned only after visitAllVersions has validated the version
* structure. Leftover prune marks read normally (the length
* accessors strip the mark bit), so only this dedicated scan can
* detect and report them - see issue #290.
*/
final int marked = MVV.countMarkedVersions(bytes, offset, length);
if (marked > 0) {
_counters._markedVersionCount += marked;
_markedVersionFaultCount++;
addFault("MVV has " + plural(marked, "version") + " with a leftover mark bit", _currentPage, 0,
foundAt);
}
if (_versionVisitor._count > 0) {
_counters._mvvCount++;
final int voffset = _versionVisitor._lastOffset;
Expand Down Expand Up @@ -289,7 +309,18 @@ protected void runTask() {
postMessage("Total " + toString(), LOG_NORMAL);
}
if (_pruneAndClear) {
if (_faults.isEmpty() && _counters._mvvPageCount == _counters._prunedPageCount
/*
* Leftover mark-bit faults (issue #290) identify volumes
* affected by a pre-#286 interrupted prune but do not block
* clearing the TransactionIndex: the prune pass above has
* already rewritten those marks (issue #292). Only other
* faults and incomplete pruning count as failures here. The
* comparison uses fault counts, not the _faults list: the
* list is capped at MAX_FAULTS, so a genuine fault found
* after marked-version faults fill the cap would be missing
* from the list but must still block the clear.
*/
if (_faultCount == _markedVersionFaultCount && _counters._mvvPageCount == _counters._prunedPageCount
&& _counters._pruningErrorCount == 0) {
final int count = _persistit.getTransactionIndex().resetMVVCounts(startTimestamp);
postMessage(String.format("%,d aborted transactions were cleared by pruning", count), LOG_NORMAL);
Expand Down Expand Up @@ -331,13 +362,15 @@ private String plural(final int n, final String m) {

private void addFault(final String description, final long page, final int level, final int position) {
final Fault fault = new Fault(resourceName(), this, description, page, _treeDepth, level, position);
_faultCount++;
if (_faults.size() < MAX_FAULTS)
_faults.add(fault);
postMessage(fault.toString(), LOG_VERBOSE);
}

private void addGarbageFault(final String description, final long page, final int level, final int position) {
final Fault fault = new Fault(resourceName(), this, description, page, 3, level, position);
_faultCount++;
if (_faults.size() < MAX_FAULTS)
_faults.add(fault);
postMessage(fault.toString(), LOG_VERBOSE);
Expand Down Expand Up @@ -544,6 +577,16 @@ public long getMvvAntiValues() {
return _counters._mvvAntiValues;
}

/**
* @return Count of MVV versions whose length field still carries a
* leftover prune mark bit, left behind by a prune interrupted
* before the issue #286 fix. Such versions read normally but
* indicate the volume was corrupted; pruning rewrites them.
*/
public long getMarkedVersionCount() {
return _counters._markedVersionCount;
}

/**
* @return Count of pages for which an expected index pointer is missing
*/
Expand Down Expand Up @@ -1144,6 +1187,7 @@ private Buffer walkRight(final int level, final long toPage, final Key key, fina
}

private boolean verifyPage(final Buffer buffer, final long page, final int level, final Key key, final Tree tree) {
_currentPage = page;
if (buffer.getPageAddress() != page) {
addFault("Buffer contains wrong page " + buffer.getPageAddress(), page, level, 0);
return false;
Expand Down
31 changes: 31 additions & 0 deletions persistit/core/src/main/java/com/persistit/MVV.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -654,6 +654,37 @@ static boolean verify(final byte[] bytes, final int offset, final int length) {
return true;
}

/**
* Count versions whose length field still carries the mark bit. Marks are
* transient state private to {@link #prune} and must never be observable
* outside it; a non-zero count on a stored MVV means the value was
* corrupted by a prune interrupted before the issue #286 fix. The length
* accessors strip the mark bit silently, so such versions read normally
* and only this scan can detect them (issue #290).
*
* @param bytes
* the byte array
* @param offset
* the index of the first byte of the MVV within the byte array
* @param length
* the count of bytes in the MVV
* @return the number of marked versions, 0 for a non-MVV value
*/
static int countMarkedVersions(final byte[] bytes, final int offset, final int length) {
if (!isArrayMVV(bytes, offset, length)) {
return 0;
}
int marked = 0;
int from = offset + 1;
while (from + LENGTH_PER_VERSION <= offset + length) {
if (isMarked(bytes, from)) {
marked++;
}
from += getLength(bytes, from) + LENGTH_PER_VERSION;
}
return marked;
}

static boolean verify(final TransactionIndex ti, final byte[] bytes, final int offset, final int length) {
if (!isArrayMVV(bytes, offset, length)) {
/*
Expand Down
Loading
Loading