From 4f90770191f0bf8f085e62b74e2dc7617a86b1af Mon Sep 17 00:00:00 2001 From: James Hilliard Date: Mon, 27 Jul 2026 19:58:11 +0200 Subject: [PATCH] metaslab: Repair overlapping space map entries automatically A duplicate or partially overlapping space map entry currently trips a range tree assertion while loading a metaslab, making the pool unusable. Add all-or-nothing range-tree operations and use them when replaying metaslab space maps. If an operation conflicts with reconstructed state, keep its complete range allocated for the rest of replay. Reconstruct invalid allocation summaries and condense repaired maps into a canonical conservative representation. Account metaslabs with invalid allocation summaries as fully allocated until they can be reconstructed, and schedule them for repair during import. Defer repair until pending TXG and log-space-map state has drained, then retry it explicitly. Keep transient I/O errors retryable and quarantine metaslabs whose maps cannot be reconstructed safely. Reject malformed entries during repair replay while preserving strict assertions for normal space-map consumers. Record completed repairs in pool history and expose aggregate repair kstats. Make trim, initialize, and RAID-Z expansion paths handle load failures without asserting or reporting false completion. Add unit coverage for the checked range-tree operations and zhack/ZTS coverage for duplicate, partial, chained, summary, pending-state, strict-replay, malformed-entry quarantine, and data-integrity cases. This cannot restore blocks overwritten before detection, but prevents ambiguous ranges from being reused. Signed-off-by: James Hilliard --- cmd/zhack.c | 244 ++++++++++- include/sys/metaslab_impl.h | 19 + include/sys/range_tree.h | 10 + include/sys/space_map.h | 13 + module/zfs/metaslab.c | 387 +++++++++++++++++- module/zfs/range_tree.c | 115 ++++-- module/zfs/spa_log_spacemap.c | 25 +- module/zfs/space_map.c | 165 +++++++- module/zfs/vdev_initialize.c | 21 +- module/zfs/vdev_raidz.c | 8 +- module/zfs/vdev_trim.c | 41 +- tests/runfiles/common.run | 3 +- tests/unit/.gitignore | 1 + tests/unit/Makefile.am | 15 + tests/unit/test_range_tree.c | 146 +++++++ tests/zfs-tests/tests/Makefile.am | 1 + .../cli_root/zhack/zhack_metaslab_repair.ksh | 185 +++++++++ 17 files changed, 1296 insertions(+), 103 deletions(-) create mode 100644 tests/unit/test_range_tree.c create mode 100755 tests/zfs-tests/tests/functional/cli_root/zhack/zhack_metaslab_repair.ksh diff --git a/cmd/zhack.c b/cmd/zhack.c index e9306e63f4c3..96087f9a8b34 100644 --- a/cmd/zhack.c +++ b/cmd/zhack.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -112,7 +113,11 @@ usage(void) " : path to vdev\n" "\n" " metaslab leak \n" - " apply allocation map from zdb to specified pool\n"); + " apply allocation map from zdb to specified pool\n" + " metaslab corrupt \n" + " inject duplicate-free, partial-free, duplicate-alloc,\n" + " partial-alloc, chained, invalid-summary, or\n" + " invalid-entry damage\n"); exit(1); } @@ -785,6 +790,241 @@ zhack_do_metaslab_leak(int argc, char **argv) spa_close(spa, FTAG); } +typedef enum zhack_metaslab_corruption { + ZHACK_METASLAB_DUPLICATE_FREE, + ZHACK_METASLAB_PARTIAL_FREE, + ZHACK_METASLAB_DUPLICATE_ALLOC, + ZHACK_METASLAB_PARTIAL_ALLOC, + ZHACK_METASLAB_CHAINED, + ZHACK_METASLAB_INVALID_SUMMARY, + ZHACK_METASLAB_INVALID_ENTRY +} zhack_metaslab_corruption_t; + +typedef struct zhack_metaslab_corrupt_arg { + zhack_metaslab_corruption_t zmca_type; + const char *zmca_name; + boolean_t zmca_injected; + uint64_t zmca_vdev; + uint64_t zmca_metaslab; + uint64_t zmca_offset; + uint64_t zmca_size; +} zhack_metaslab_corrupt_arg_t; + +static boolean_t +zhack_metaslab_corrupt_range(metaslab_t *msp, + zhack_metaslab_corruption_t type, uint64_t *offset, uint64_t *size) +{ + zfs_range_tree_t *rt = msp->ms_allocatable; + uint64_t unit = 1ULL << msp->ms_sm->sm_shift; + uint64_t cursor = msp->ms_start; + zfs_btree_index_t where; + zfs_range_seg_t *rs; + + for (rs = zfs_btree_first(&rt->rt_root, &where); rs != NULL; + rs = zfs_btree_next(&rt->rt_root, &where, &where)) { + uint64_t start = zfs_rs_get_start(rs, rt); + uint64_t end = zfs_rs_get_end(rs, rt); + + if ((type == ZHACK_METASLAB_DUPLICATE_ALLOC) && + start - cursor >= unit) { + *offset = cursor; + *size = unit; + return (B_TRUE); + } + if (type == ZHACK_METASLAB_DUPLICATE_FREE && + end - start >= unit) { + *offset = start; + *size = unit; + return (B_TRUE); + } + if ((type == ZHACK_METASLAB_PARTIAL_FREE || + type == ZHACK_METASLAB_PARTIAL_ALLOC) && + start - msp->ms_start >= unit) { + *offset = start - unit; + *size = 2 * unit; + return (B_TRUE); + } + if (type == ZHACK_METASLAB_CHAINED && end - start >= 2 * unit) { + *offset = start; + *size = 2 * unit; + return (B_TRUE); + } + if ((type == ZHACK_METASLAB_PARTIAL_FREE || + type == ZHACK_METASLAB_PARTIAL_ALLOC) && + msp->ms_start + msp->ms_size - end >= unit) { + *offset = end - unit; + *size = 2 * unit; + return (B_TRUE); + } + cursor = end; + } + + if (type == ZHACK_METASLAB_DUPLICATE_ALLOC && + msp->ms_start + msp->ms_size - cursor >= unit) { + *offset = cursor; + *size = unit; + return (B_TRUE); + } + + return (B_FALSE); +} + +static void +zhack_metaslab_corrupt_invalid_entry(space_map_t *sm, dmu_tx_t *tx) +{ + uint64_t entry = SM_OFFSET_ENCODE(sm->sm_size >> sm->sm_shift) | + SM_TYPE_ENCODE(SM_ALLOC) | SM_RUN_ENCODE(1); + + dmu_buf_will_dirty(sm->sm_dbuf, tx); + dmu_write(sm->sm_os, space_map_object(sm), space_map_length(sm), + sizeof (entry), &entry, tx, DMU_READ_NO_PREFETCH); + sm->sm_phys->smp_length += sizeof (entry); +} + +static void +zhack_metaslab_corrupt_sync(void *arg, dmu_tx_t *tx) +{ + zhack_metaslab_corrupt_arg_t *zmca = arg; + spa_t *spa = dmu_tx_pool(tx)->dp_spa; + vdev_t *rvd = spa->spa_root_vdev; + + for (uint64_t c = 0; c < rvd->vdev_children; c++) { + vdev_t *vd = rvd->vdev_child[c]; + if (vd->vdev_mg == NULL) + continue; + + for (uint64_t m = 0; m < vd->vdev_ms_count; m++) { + metaslab_t *msp = vd->vdev_ms[m]; + + mutex_enter(&msp->ms_lock); + if (msp->ms_sm == NULL || metaslab_load(msp) != 0) { + mutex_exit(&msp->ms_lock); + continue; + } + + uint64_t offset = 0; + uint64_t size = 0; + boolean_t invalid_summary = zmca->zmca_type == + ZHACK_METASLAB_INVALID_SUMMARY; + boolean_t invalid_entry = zmca->zmca_type == + ZHACK_METASLAB_INVALID_ENTRY; + if (invalid_summary) { + dmu_buf_will_dirty(msp->ms_sm->sm_dbuf, tx); + msp->ms_sm->sm_phys->smp_alloc = + -(int64_t)(1ULL << msp->ms_sm->sm_shift); + } else if (invalid_entry) { + offset = msp->ms_start + msp->ms_size; + size = 1ULL << msp->ms_sm->sm_shift; + } else if (!zhack_metaslab_corrupt_range(msp, + zmca->zmca_type, &offset, &size)) { + mutex_exit(&msp->ms_lock); + continue; + } + + space_map_t *sm = msp->ms_sm; + zfs_range_tree_t *rt = NULL; + if (!invalid_summary && !invalid_entry) { + rt = zfs_range_tree_create(NULL, + ZFS_RANGE_SEG64, NULL, 0, 0); + uint64_t first_size = + zmca->zmca_type == ZHACK_METASLAB_CHAINED ? + size / 2 : size; + zfs_range_tree_add(rt, offset, first_size); + } + mutex_exit(&msp->ms_lock); + + if (invalid_entry) { + zhack_metaslab_corrupt_invalid_entry(sm, tx); + } else if (rt != NULL) { + maptype_t maptype = + zmca->zmca_type == + ZHACK_METASLAB_DUPLICATE_ALLOC || + zmca->zmca_type == + ZHACK_METASLAB_PARTIAL_ALLOC ? + SM_ALLOC : SM_FREE; + space_map_write(sm, rt, maptype, + SM_NO_VDEVID, tx); + if (zmca->zmca_type == ZHACK_METASLAB_CHAINED) { + zfs_range_tree_vacate(rt, NULL, NULL); + zfs_range_tree_add(rt, offset, size); + space_map_write(sm, rt, SM_ALLOC, + SM_NO_VDEVID, tx); + } + zfs_range_tree_vacate(rt, NULL, NULL); + zfs_range_tree_destroy(rt); + } + + zmca->zmca_injected = B_TRUE; + zmca->zmca_vdev = vd->vdev_id; + zmca->zmca_metaslab = msp->ms_id; + zmca->zmca_offset = offset; + zmca->zmca_size = size; + return; + } + } +} + +static void +zhack_do_metaslab_corrupt(int argc, char **argv) +{ + char *target; + spa_t *spa; + + argc--; + argv++; + if (argc != 2) { + (void) fprintf(stderr, + "error: expected a pool name and corruption type\n"); + usage(); + } + target = argv[0]; + + zhack_metaslab_corrupt_arg_t zmca = { + .zmca_name = argv[1], + }; + if (strcmp(zmca.zmca_name, "duplicate-free") == 0) { + zmca.zmca_type = ZHACK_METASLAB_DUPLICATE_FREE; + } else if (strcmp(zmca.zmca_name, "partial-free") == 0) { + zmca.zmca_type = ZHACK_METASLAB_PARTIAL_FREE; + } else if (strcmp(zmca.zmca_name, "duplicate-alloc") == 0) { + zmca.zmca_type = ZHACK_METASLAB_DUPLICATE_ALLOC; + } else if (strcmp(zmca.zmca_name, "partial-alloc") == 0) { + zmca.zmca_type = ZHACK_METASLAB_PARTIAL_ALLOC; + } else if (strcmp(zmca.zmca_name, "chained") == 0) { + zmca.zmca_type = ZHACK_METASLAB_CHAINED; + } else if (strcmp(zmca.zmca_name, "invalid-summary") == 0) { + zmca.zmca_type = ZHACK_METASLAB_INVALID_SUMMARY; + } else if (strcmp(zmca.zmca_name, "invalid-entry") == 0) { + zmca.zmca_type = ZHACK_METASLAB_INVALID_ENTRY; + } else { + (void) fprintf(stderr, "error: unknown corruption type: %s\n", + zmca.zmca_name); + usage(); + } + + /* + * Committing an intentionally inconsistent space map can make its + * allocation summary disagree with the loaded metaslab. Leave other + * debug checks enabled while allowing the injector to close the pool. + */ + zfs_flags &= ~ZFS_DEBUG_METASLAB_VERIFY; + zhack_spa_open(target, B_FALSE, FTAG, &spa); + + VERIFY0(dsl_sync_task(spa_name(spa), NULL, + zhack_metaslab_corrupt_sync, &zmca, 5, + ZFS_SPACE_CHECK_NONE)); + if (!zmca.zmca_injected) { + fatal(spa, FTAG, "no suitable metaslab range available in '%s'", + target); + } + + (void) printf("injected %s: vdev=%" PRIu64 + " metaslab=%" PRIu64 " offset=%" PRIu64 " size=%" PRIu64 "\n", + zmca.zmca_name, zmca.zmca_vdev, zmca.zmca_metaslab, + zmca.zmca_offset, zmca.zmca_size); + spa_close(spa, FTAG); +} + static int zhack_do_metaslab(int argc, char **argv) { @@ -801,6 +1041,8 @@ zhack_do_metaslab(int argc, char **argv) subcommand = argv[0]; if (strcmp(subcommand, "leak") == 0) { zhack_do_metaslab_leak(argc, argv); + } else if (strcmp(subcommand, "corrupt") == 0) { + zhack_do_metaslab_corrupt(argc, argv); } else { (void) fprintf(stderr, "error: unknown subcommand: %s\n", subcommand); diff --git a/include/sys/metaslab_impl.h b/include/sys/metaslab_impl.h index 78b56529a399..92dbeca75c39 100644 --- a/include/sys/metaslab_impl.h +++ b/include/sys/metaslab_impl.h @@ -338,6 +338,19 @@ struct metaslab_group { * ensure that allocations are not performed on the metaslab that is * being written. */ +typedef enum metaslab_load_state { + METASLAB_LOAD_NORMAL = 0, + METASLAB_LOAD_DEFERRED, + METASLAB_LOAD_UNLOADABLE, +} metaslab_load_state_t; + +typedef struct metaslab_repair { + int64_t msr_smp_alloc; + uint64_t msr_allocated; + uint64_t msr_entries; + uint64_t msr_bytes; +} metaslab_repair_t; + struct metaslab { /* * This is the main lock of the metaslab and its purpose is to @@ -404,6 +417,12 @@ struct metaslab { boolean_t ms_condensing; /* condensing? */ boolean_t ms_condense_wanted; + /* Details retained until a repaired space map is condensed. */ + metaslab_repair_t ms_repair; + /* The on-disk smp_alloc is outside the metaslab's valid range. */ + boolean_t ms_smp_alloc_invalid; + metaslab_load_state_t ms_load_state; + /* * The number of consumers which have disabled the metaslab. */ diff --git a/include/sys/range_tree.h b/include/sys/range_tree.h index 0f6def36f9f6..8c0fe129c48f 100644 --- a/include/sys/range_tree.h +++ b/include/sys/range_tree.h @@ -310,7 +310,17 @@ uint64_t zfs_range_tree_max(zfs_range_tree_t *rt); uint64_t zfs_range_tree_span(zfs_range_tree_t *rt); void zfs_range_tree_add(void *arg, uint64_t start, uint64_t size); +/* + * For gap-free trees, try_add() succeeds only when the requested range does + * not overlap the tree, and try_remove() succeeds only when one segment + * contains the entire range. Both return B_FALSE without modifying the tree + * otherwise. + */ +boolean_t zfs_range_tree_try_add(zfs_range_tree_t *rt, uint64_t start, + uint64_t size); void zfs_range_tree_remove(void *arg, uint64_t start, uint64_t size); +boolean_t zfs_range_tree_try_remove(zfs_range_tree_t *rt, uint64_t start, + uint64_t size); void zfs_range_tree_remove_fill(zfs_range_tree_t *rt, uint64_t start, uint64_t size); void zfs_range_tree_adjust_fill(zfs_range_tree_t *rt, zfs_range_seg_t *rs, diff --git a/include/sys/space_map.h b/include/sys/space_map.h index 6363f347c591..04c6795e9ed7 100644 --- a/include/sys/space_map.h +++ b/include/sys/space_map.h @@ -208,9 +208,22 @@ boolean_t sm_entry_is_double_word(uint64_t e); typedef int (*sm_cb_t)(space_map_entry_t *sme, void *arg); +typedef struct space_map_load_result { + /* Allocated bytes reconstructed from the entries that were loaded. */ + uint64_t smlr_allocated; + uint64_t smlr_repaired_entries; + uint64_t smlr_affected_bytes; + space_map_entry_t smlr_first_entry; + space_map_entry_t smlr_last_entry; + uint64_t smlr_first_free_bytes; + uint64_t smlr_last_free_bytes; +} space_map_load_result_t; + int space_map_load(space_map_t *sm, zfs_range_tree_t *rt, maptype_t maptype); int space_map_load_length(space_map_t *sm, zfs_range_tree_t *rt, maptype_t maptype, uint64_t length); +int space_map_load_length_repair(space_map_t *sm, zfs_range_tree_t *rt, + uint64_t length, space_map_load_result_t *result); int space_map_iterate(space_map_t *sm, uint64_t length, sm_cb_t callback, void *arg); int space_map_incremental_destroy(space_map_t *sm, sm_cb_t callback, void *arg, diff --git a/module/zfs/metaslab.c b/module/zfs/metaslab.c index f29bdc0118b8..e634939f9136 100644 --- a/module/zfs/metaslab.c +++ b/module/zfs/metaslab.c @@ -366,6 +366,10 @@ typedef struct metaslab_stats { kstat_named_t metaslabstat_reload_tree; kstat_named_t metaslabstat_too_many_tries; kstat_named_t metaslabstat_try_hard; + kstat_named_t metaslabstat_load_repairs; + kstat_named_t metaslabstat_load_repaired_entries; + kstat_named_t metaslabstat_load_repaired_bytes; + kstat_named_t metaslabstat_load_unloadable; } metaslab_stats_t; static metaslab_stats_t metaslab_stats = { @@ -373,10 +377,16 @@ static metaslab_stats_t metaslab_stats = { { "reload_tree", KSTAT_DATA_UINT64 }, { "too_many_tries", KSTAT_DATA_UINT64 }, { "try_hard", KSTAT_DATA_UINT64 }, + { "load_repairs", KSTAT_DATA_UINT64 }, + { "load_repaired_entries", KSTAT_DATA_UINT64 }, + { "load_repaired_bytes", KSTAT_DATA_UINT64 }, + { "load_unloadable", KSTAT_DATA_UINT64 }, }; #define METASLABSTAT_BUMP(stat) \ atomic_inc_64(&metaslab_stats.stat.value.ui64); +#define METASLABSTAT_ADD(stat, amount) \ + atomic_add_64(&metaslab_stats.stat.value.ui64, amount); char * metaslab_rt_name(metaslab_group_t *mg, metaslab_t *ms, const char *name) @@ -2159,20 +2169,31 @@ metaslab_verify_space(metaslab_t *msp, uint64_t txg) !msp->ms_loaded) return; - /* - * Even though the smp_alloc field can get negative, - * when it comes to a metaslab's space map, that should - * never be the case. - */ - ASSERT3S(space_map_allocated(msp->ms_sm), >=, 0); - - ASSERT3U(space_map_allocated(msp->ms_sm), >=, + uint64_t expected_allocated; + if (msp->ms_repair.msr_entries != 0 || + msp->ms_smp_alloc_invalid) { + expected_allocated = msp->ms_repair.msr_allocated; + } else { + /* + * Even though smp_alloc can be negative for other space + * maps, a healthy metaslab space map must be in range. + */ + ASSERT3S(space_map_allocated(msp->ms_sm), >=, 0); + ASSERT3U(space_map_allocated(msp->ms_sm), <=, msp->ms_size); + expected_allocated = + (uint64_t)space_map_allocated(msp->ms_sm); + } + ASSERT3U(expected_allocated, >=, zfs_range_tree_space(msp->ms_unflushed_frees)); - - ASSERT3U(metaslab_allocated_space(msp), ==, - space_map_allocated(msp->ms_sm) + - zfs_range_tree_space(msp->ms_unflushed_allocs) - + ASSERT3U(expected_allocated, <=, UINT64_MAX - + zfs_range_tree_space(msp->ms_unflushed_allocs)); + expected_allocated += + zfs_range_tree_space(msp->ms_unflushed_allocs); + ASSERT3U(expected_allocated, >=, zfs_range_tree_space(msp->ms_unflushed_frees)); + expected_allocated -= + zfs_range_tree_space(msp->ms_unflushed_frees); + ASSERT3U(metaslab_allocated_space(msp), ==, expected_allocated); sm_free_space = msp->ms_size - metaslab_allocated_space(msp); @@ -2199,6 +2220,119 @@ metaslab_verify_space(metaslab_t *msp, uint64_t txg) VERIFY3U(sm_free_space, ==, msp_free_space); } +static int +metaslab_reconcile_load_repair(metaslab_t *msp, + const space_map_load_result_t *result, boolean_t repair_needed, + int64_t *alloc_delta) +{ + uint64_t old_allocated = msp->ms_allocated_space; + uint64_t new_allocated = repair_needed ? + result->smlr_allocated : old_allocated; + + if (new_allocated > msp->ms_size) + return (SET_ERROR(ECKSUM)); + + uint64_t magnitude; + if (new_allocated >= old_allocated) { + magnitude = new_allocated - old_allocated; + if (magnitude > INT64_MAX) + return (SET_ERROR(ECKSUM)); + *alloc_delta = (int64_t)magnitude; + } else { + magnitude = old_allocated - new_allocated; + if (magnitude > INT64_MAX) + return (SET_ERROR(ECKSUM)); + *alloc_delta = -(int64_t)magnitude; + } + + msp->ms_allocated_space = new_allocated; + return (0); +} + +static void +metaslab_repair_clear(metaslab_t *msp) +{ + memset(&msp->ms_repair, 0, sizeof (msp->ms_repair)); +} + +static boolean_t +metaslab_load_has_pending_changes(metaslab_t *msp) +{ + if (msp->ms_allocating_total != 0 || + msp->ms_allocated_this_txg != 0 || + msp->ms_deferspace != 0 || + !zfs_range_tree_is_empty(msp->ms_unflushed_allocs) || + !zfs_range_tree_is_empty(msp->ms_unflushed_frees) || + !zfs_range_tree_is_empty(msp->ms_freeing) || + !zfs_range_tree_is_empty(msp->ms_freed) || + !zfs_range_tree_is_empty(msp->ms_checkpointing)) + return (B_TRUE); + + for (int t = 0; t < TXG_SIZE; t++) { + if (!zfs_range_tree_is_empty(msp->ms_allocating[t])) + return (B_TRUE); + } + for (int t = 0; t < TXG_DEFER_SIZE; t++) { + if (!zfs_range_tree_is_empty(msp->ms_defer[t])) + return (B_TRUE); + } + + return (B_FALSE); +} + +static void +metaslab_account_unloadable(metaslab_t *msp) +{ + ASSERT(MUTEX_HELD(&msp->ms_lock)); + ASSERT3U(msp->ms_allocated_space, <=, msp->ms_size); + + /* + * Once all known changes have been accounted, charge the remaining + * capacity as allocated so an unreadable metaslab is never reported as + * usable space. + */ + uint64_t unavailable = msp->ms_size - msp->ms_allocated_space; + while (unavailable != 0) { + int64_t delta = (int64_t)MIN(unavailable, + (uint64_t)INT64_MAX); + metaslab_space_update(msp->ms_group, delta, 0, 0); + unavailable -= (uint64_t)delta; + } + msp->ms_allocated_space = msp->ms_size; +} + +static void +metaslab_mark_unloadable(metaslab_t *msp) +{ + ASSERT(MUTEX_HELD(&msp->ms_lock)); + + boolean_t pending = metaslab_load_has_pending_changes(msp); + if (!pending) + metaslab_account_unloadable(msp); + + metaslab_repair_clear(msp); + msp->ms_load_state = METASLAB_LOAD_UNLOADABLE; + msp->ms_condense_wanted = B_FALSE; + msp->ms_max_size = 0; + zfs_range_tree_vacate(msp->ms_trim, NULL, NULL); + metaslab_group_sort(msp->ms_group, msp, 0); + METASLABSTAT_BUMP(metaslabstat_load_unloadable); + + spa_t *spa = msp->ms_group->mg_vd->vdev_spa; + uint64_t txg = spa_syncing_txg(spa); + if (pending && spa_writeable(spa) && + txg < spa_final_dirty_txg(spa)) { + vdev_dirty(msp->ms_group->mg_vd, VDD_METASLAB, msp, txg + 1); + } + + zfs_dbgmsg("metaslab_load: quarantining unloadable metaslab, " + "spa %s, vdev_id %llu, ms_id %llu, object %llu", + spa_name(msp->ms_group->mg_vd->vdev_spa), + (u_longlong_t)msp->ms_group->mg_vd->vdev_id, + (u_longlong_t)msp->ms_id, + (u_longlong_t)space_map_object(msp->ms_sm)); +} + static void metaslab_aux_histograms_clear(metaslab_t *msp) { @@ -2334,6 +2468,9 @@ metaslab_verify_weight_and_frag(metaslab_t *msp) if (msp->ms_group == NULL) return; + if (msp->ms_load_state != METASLAB_LOAD_NORMAL) + return; + /* * Devices being removed always return a weight of 0 and leave * fragmentation and ms_max_size as is - there is nothing for @@ -2495,6 +2632,7 @@ static int metaslab_load_impl(metaslab_t *msp) { int error = 0; + space_map_load_result_t load_result = { 0 }; ASSERT(MUTEX_HELD(&msp->ms_lock)); ASSERT(msp->ms_loading); @@ -2540,8 +2678,8 @@ metaslab_load_impl(metaslab_t *msp) mrap->mra_floor_shift = metaslab_by_size_min_shift; if (msp->ms_sm != NULL) { - error = space_map_load_length(msp->ms_sm, msp->ms_allocatable, - SM_FREE, length); + error = space_map_load_length_repair(msp->ms_sm, + msp->ms_allocatable, length, &load_result); /* Now, populate the size-sorted tree. */ metaslab_rt_create(msp->ms_allocatable, mrap); @@ -2603,7 +2741,108 @@ metaslab_load_impl(metaslab_t *msp) } ASSERT3P(msp->ms_group, !=, NULL); + metaslab_group_t *mg = msp->ms_group; + vdev_t *vd = mg->mg_vd; + spa_t *spa = vd->vdev_spa; + int64_t alloc_delta; + boolean_t repair_needed = load_result.smlr_repaired_entries != 0 || + msp->ms_smp_alloc_invalid; + + /* + * Repair is only self-contained when no changes outside the loaded + * space map must be merged into ms_allocatable. If there are pending + * changes, defer the load until syncing drains them rather than risk + * making an ambiguous range allocatable again. + */ + if (repair_needed && metaslab_load_has_pending_changes(msp)) { + zfs_dbgmsg("metaslab_load: cannot safely persist overlap " + "repair with pending changes, spa %s, vdev_id %llu, " + "ms_id %llu, object %llu", + spa_name(spa), (u_longlong_t)vd->vdev_id, + (u_longlong_t)msp->ms_id, + (u_longlong_t)space_map_object(msp->ms_sm)); + zfs_range_tree_vacate(msp->ms_allocatable, NULL, NULL); + msp->ms_load_state = METASLAB_LOAD_DEFERRED; + msp->ms_condense_wanted = B_FALSE; + metaslab_group_sort(mg, msp, 0); + uint64_t txg = spa_syncing_txg(spa); + if (spa_writeable(spa) && txg < spa_final_dirty_txg(spa)) + vdev_dirty(vd, VDD_METASLAB, msp, txg + 1); + mutex_exit(&msp->ms_sync_lock); + return (SET_ERROR(EAGAIN)); + } + + error = metaslab_reconcile_load_repair(msp, &load_result, + repair_needed, &alloc_delta); + if (error != 0) { + zfs_range_tree_vacate(msp->ms_allocatable, NULL, NULL); + mutex_exit(&msp->ms_sync_lock); + return (error); + } + msp->ms_loaded = B_TRUE; + msp->ms_load_state = METASLAB_LOAD_NORMAL; + if (alloc_delta != 0) + metaslab_space_update(mg, alloc_delta, 0, 0); + + if (repair_needed) { + msp->ms_repair = (metaslab_repair_t) { + .msr_smp_alloc = space_map_allocated(msp->ms_sm), + .msr_allocated = load_result.smlr_allocated, + .msr_entries = load_result.smlr_repaired_entries, + .msr_bytes = load_result.smlr_affected_bytes, + }; + + uint64_t txg = spa_syncing_txg(spa); + if (spa_writeable(spa) && txg < spa_final_dirty_txg(spa)) { + msp->ms_condense_wanted = B_TRUE; + vdev_dirty(vd, VDD_METASLAB, msp, txg + 1); + } + + METASLABSTAT_BUMP(metaslabstat_load_repairs); + METASLABSTAT_ADD(metaslabstat_load_repaired_entries, + load_result.smlr_repaired_entries); + METASLABSTAT_ADD(metaslabstat_load_repaired_bytes, + load_result.smlr_affected_bytes); + zfs_dbgmsg("metaslab_load: repaired invalid_smp_alloc=%u and " + "quarantined %llu overlapping space map entries " + "affecting %llu bytes, smp_alloc %lld, reconstructed " + "allocation %llu, spa %s, vdev_id %llu, ms_id %llu, " + "object %llu", + msp->ms_smp_alloc_invalid, + (u_longlong_t)load_result.smlr_repaired_entries, + (u_longlong_t)load_result.smlr_affected_bytes, + (longlong_t)msp->ms_repair.msr_smp_alloc, + (u_longlong_t)load_result.smlr_allocated, spa_name(spa), + (u_longlong_t)vd->vdev_id, (u_longlong_t)msp->ms_id, + (u_longlong_t)space_map_object(msp->ms_sm)); + if (load_result.smlr_repaired_entries != 0) { + zfs_dbgmsg("metaslab_load: first %s offset=%llx " + "run=%llx free_before=%llx txg=%llu pass=%llu; " + "last %s offset=%llx run=%llx free_before=%llx " + "txg=%llu pass=%llu", + load_result.smlr_first_entry.sme_type == SM_ALLOC ? + "ALLOC" : "FREE", + (u_longlong_t) + load_result.smlr_first_entry.sme_offset, + (u_longlong_t) + load_result.smlr_first_entry.sme_run, + (u_longlong_t)load_result.smlr_first_free_bytes, + (u_longlong_t) + load_result.smlr_first_entry.sme_txg, + (u_longlong_t) + load_result.smlr_first_entry.sme_sync_pass, + load_result.smlr_last_entry.sme_type == SM_ALLOC ? + "ALLOC" : "FREE", + (u_longlong_t) + load_result.smlr_last_entry.sme_offset, + (u_longlong_t)load_result.smlr_last_entry.sme_run, + (u_longlong_t)load_result.smlr_last_free_bytes, + (u_longlong_t)load_result.smlr_last_entry.sme_txg, + (u_longlong_t) + load_result.smlr_last_entry.sme_sync_pass); + } + } /* * Apply all the unflushed changes to ms_allocatable right @@ -2616,7 +2855,6 @@ metaslab_load_impl(metaslab_t *msp) zfs_range_tree_add, msp->ms_allocatable); ASSERT3P(msp->ms_group, !=, NULL); - spa_t *spa = msp->ms_group->mg_vd->vdev_spa; if (spa_syncing_log_sm(spa) != NULL) { ASSERT(spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP)); @@ -2679,10 +2917,12 @@ metaslab_load_impl(metaslab_t *msp) uint64_t weight = msp->ms_weight; uint64_t max_size = msp->ms_max_size; metaslab_recalculate_weight_and_sort(msp); - if (!WEIGHT_IS_SPACEBASED(weight)) + if (!WEIGHT_IS_SPACEBASED(weight) && + load_result.smlr_repaired_entries == 0) ASSERT3U(weight, <=, msp->ms_weight); msp->ms_max_size = metaslab_largest_allocatable(msp); - ASSERT3U(max_size, <=, msp->ms_max_size); + if (load_result.smlr_repaired_entries == 0) + ASSERT3U(max_size, <=, msp->ms_max_size); hrtime_t load_end = gethrtime(); msp->ms_load_time = load_end; zfs_dbgmsg("metaslab_load: txg %llu, spa %s, class %s, vdev_id %llu, " @@ -2725,6 +2965,10 @@ metaslab_load(metaslab_t *msp) metaslab_load_wait(msp); if (msp->ms_loaded) return (0); + if (msp->ms_load_state == METASLAB_LOAD_DEFERRED) + return (SET_ERROR(EAGAIN)); + if (msp->ms_load_state == METASLAB_LOAD_UNLOADABLE) + return (SET_ERROR(ECKSUM)); VERIFY(!msp->ms_loading); ASSERT(!msp->ms_condensing); @@ -2764,6 +3008,8 @@ metaslab_load(metaslab_t *msp) int error = metaslab_load_impl(msp); ASSERT(MUTEX_HELD(&msp->ms_lock)); + if (error == ECKSUM) + metaslab_mark_unloadable(msp); msp->ms_loading = B_FALSE; cv_broadcast(&msp->ms_load_cv); @@ -2942,7 +3188,20 @@ metaslab_init(metaslab_group_t *mg, uint64_t id, uint64_t object, } ASSERT(ms->ms_sm != NULL); - ms->ms_allocated_space = space_map_allocated(ms->ms_sm); + int64_t allocated = space_map_allocated(ms->ms_sm); + if (allocated < 0 || (uint64_t)allocated > ms->ms_size) { + ms->ms_allocated_space = ms->ms_size; + ms->ms_smp_alloc_invalid = B_TRUE; + zfs_dbgmsg("metaslab_init: invalid allocated space " + "%lld in spa %s, vdev_id %llu, ms_id %llu, " + "object %llu", + (longlong_t)allocated, spa_name(spa), + (u_longlong_t)vd->vdev_id, + (u_longlong_t)ms->ms_id, + (u_longlong_t)space_map_object(ms->ms_sm)); + } else { + ms->ms_allocated_space = (uint64_t)allocated; + } } uint64_t shift, start; @@ -3004,6 +3263,18 @@ metaslab_init(metaslab_group_t *mg, uint64_t id, uint64_t object, metaslab_space_update(mg, metaslab_allocated_space(ms), 0, 0); } + /* + * The conservative accounting above gives this metaslab zero weight, + * so explicitly schedule a preload to reconstruct its allocation count. + */ + if (ms->ms_smp_alloc_invalid && spa_writeable(spa)) { + uint64_t repair_txg = spa_syncing_txg(spa); + if (repair_txg < spa_final_dirty_txg(spa)) { + ms->ms_condense_wanted = B_TRUE; + vdev_dirty(vd, VDD_METASLAB, ms, repair_txg + 1); + } + } + if (txg != 0) { vdev_dirty(vd, 0, NULL, txg); vdev_dirty(vd, VDD_METASLAB, ms, txg); @@ -3469,6 +3740,9 @@ metaslab_segment_weight(metaslab_t *msp) static boolean_t metaslab_should_allocate(metaslab_t *msp, uint64_t asize, boolean_t try_hard) { + if (unlikely(msp->ms_load_state != METASLAB_LOAD_NORMAL)) + return (B_FALSE); + /* * This case will usually but not always get caught by the checks below; * metaslabs can be loaded by various means, including the trim and @@ -3517,6 +3791,9 @@ metaslab_weight(metaslab_t *msp, boolean_t nodirty) ASSERT(MUTEX_HELD(&msp->ms_lock)); + if (msp->ms_load_state != METASLAB_LOAD_NORMAL) + return (0); + metaslab_set_fragmentation(msp, nodirty); /* @@ -3785,8 +4062,8 @@ metaslab_preload(void *arg) ASSERT(!MUTEX_HELD(&msp->ms_group->mg_lock)); mutex_enter(&msp->ms_lock); - (void) metaslab_load(msp); - metaslab_set_selected_txg(msp, spa_syncing_txg(spa)); + if (metaslab_load(msp) == 0) + metaslab_set_selected_txg(msp, spa_syncing_txg(spa)); mutex_exit(&msp->ms_lock); spl_fstrans_unmark(cookie); } @@ -3893,6 +4170,8 @@ metaslab_condense(metaslab_t *msp, dmu_tx_t *tx) space_map_t *sm = msp->ms_sm; uint64_t txg = dmu_tx_get_txg(tx); spa_t *spa = msp->ms_group->mg_vd->vdev_spa; + metaslab_repair_t repair = msp->ms_repair; + boolean_t repair_summary = msp->ms_smp_alloc_invalid; ASSERT(MUTEX_HELD(&msp->ms_lock)); ASSERT(msp->ms_loaded); @@ -4032,10 +4311,24 @@ metaslab_condense(metaslab_t *msp, dmu_tx_t *tx) zfs_range_tree_destroy(condense_tree); zfs_range_tree_vacate(tmp_tree, NULL, NULL); zfs_range_tree_destroy(tmp_tree); + if (repair.msr_entries != 0 || repair_summary) { + spa_history_log_internal(spa, "metaslab repair", tx, + "vdev=%llu metaslab=%llu object=%llu entries=%llu " + "bytes=%llu smp_alloc=%lld reconstructed_alloc=%llu " + "invalid_smp_alloc=%u", + (u_longlong_t)msp->ms_group->mg_vd->vdev_id, + (u_longlong_t)msp->ms_id, (u_longlong_t)object, + (u_longlong_t)repair.msr_entries, + (u_longlong_t)repair.msr_bytes, + (longlong_t)repair.msr_smp_alloc, + (u_longlong_t)repair.msr_allocated, repair_summary); + } mutex_enter(&msp->ms_lock); msp->ms_condensing = B_FALSE; metaslab_flush_update(msp, tx); + metaslab_repair_clear(msp); + msp->ms_smp_alloc_invalid = B_FALSE; } static void @@ -4249,6 +4542,10 @@ metaslab_flush(metaslab_t *msp, dmu_tx_t *tx) zfs_range_tree_vacate(msp->ms_unflushed_allocs, NULL, NULL); zfs_range_tree_vacate(msp->ms_unflushed_frees, NULL, NULL); + if (msp->ms_load_state == METASLAB_LOAD_UNLOADABLE && + !metaslab_load_has_pending_changes(msp)) + metaslab_account_unloadable(msp); + metaslab_verify_space(msp, dmu_tx_get_txg(tx)); metaslab_verify_weight_and_frag(msp); @@ -4303,6 +4600,8 @@ metaslab_sync(metaslab_t *msp, uint64_t txg) if (zfs_range_tree_is_empty(alloctree) && zfs_range_tree_is_empty(msp->ms_freeing) && zfs_range_tree_is_empty(msp->ms_checkpointing) && + !(msp->ms_load_state != METASLAB_LOAD_NORMAL && + txg <= spa_final_dirty_txg(spa)) && !(msp->ms_loaded && msp->ms_condense_wanted && txg <= spa_final_dirty_txg(spa))) return; @@ -4375,6 +4674,29 @@ metaslab_sync(metaslab_t *msp, uint64_t txg) mutex_enter(&msp->ms_sync_lock); mutex_enter(&msp->ms_lock); + /* + * A repair load can be deferred by changes which still live in the + * pool-wide log space map. Force those changes into this metaslab's + * space map so the next preload can repair and condense it. + */ + if (spa_sync_pass(spa) == 1 && + msp->ms_load_state != METASLAB_LOAD_NORMAL && + spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP) && + metaslab_unflushed_txg(msp) != 0 && + metaslab_unflushed_txg(msp) < txg && + metaslab_unflushed_dirty(msp)) { + boolean_t flush_only = zfs_range_tree_is_empty(alloctree) && + zfs_range_tree_is_empty(msp->ms_freeing) && + zfs_range_tree_is_empty(msp->ms_checkpointing); + + if (metaslab_flush(msp, tx) && flush_only) { + mutex_exit(&msp->ms_lock); + mutex_exit(&msp->ms_sync_lock); + dmu_tx_commit(tx); + return; + } + } + /* * Note: metaslab_condense() clears the space map's histogram. * Therefore we must verify and remove this histogram before @@ -4575,6 +4897,7 @@ metaslab_sync_done(metaslab_t *msp, uint64_t txg) zfs_range_tree_t **defer_tree; int64_t alloc_delta, defer_delta; boolean_t defer_allowed = B_TRUE; + boolean_t retry_load = B_FALSE; ASSERT(!vd->vdev_ishole); @@ -4634,7 +4957,8 @@ metaslab_sync_done(metaslab_t *msp, uint64_t txg) * can be discarded at any time with the sole consequence of recent * frees not being trimmed. */ - if (spa_get_autotrim(spa) == SPA_AUTOTRIM_ON) { + if (spa_get_autotrim(spa) == SPA_AUTOTRIM_ON && + msp->ms_load_state != METASLAB_LOAD_UNLOADABLE) { zfs_range_tree_walk(*defer_tree, zfs_range_tree_add, msp->ms_trim); if (!defer_allowed) { @@ -4694,7 +5018,28 @@ metaslab_sync_done(metaslab_t *msp, uint64_t txg) ASSERT0(zfs_range_tree_space(msp->ms_checkpointing)); msp->ms_allocating_total -= msp->ms_allocated_this_txg; msp->ms_allocated_this_txg = 0; + if (msp->ms_load_state == METASLAB_LOAD_DEFERRED) { + if (!metaslab_load_has_pending_changes(msp)) { + msp->ms_load_state = METASLAB_LOAD_NORMAL; + metaslab_recalculate_weight_and_sort(msp); + retry_load = B_TRUE; + } else if (txg < spa_final_dirty_txg(spa)) { + vdev_dirty(vd, VDD_METASLAB, msp, txg + 1); + } + } + if (msp->ms_load_state == METASLAB_LOAD_UNLOADABLE) { + if (!metaslab_load_has_pending_changes(msp)) { + metaslab_account_unloadable(msp); + } else if (txg < spa_final_dirty_txg(spa)) { + vdev_dirty(vd, VDD_METASLAB, msp, txg + 1); + } + } mutex_exit(&msp->ms_lock); + + if (retry_load && !spa_shutting_down(spa)) { + VERIFY(taskq_dispatch(spa->spa_metaslab_taskq, + metaslab_preload, msp, TQ_SLEEP) != TASKQID_INVALID); + } } void diff --git a/module/zfs/range_tree.c b/module/zfs/range_tree.c index 916889c3130c..b937225f9783 100644 --- a/module/zfs/range_tree.c +++ b/module/zfs/range_tree.c @@ -325,8 +325,9 @@ zfs_range_tree_adjust_fill(zfs_range_tree_t *rt, zfs_range_seg_t *rs, rt->rt_ops->rtop_add(rt, rs, rt->rt_arg); } -static void -zfs_range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill) +static boolean_t +zfs_range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill, + boolean_t panic_on_error) { zfs_range_tree_t *rt = arg; zfs_btree_index_t where; @@ -356,17 +357,20 @@ zfs_range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill) uint64_t rstart = zfs_rs_get_start(rs, rt); uint64_t rend = zfs_rs_get_end(rs, rt); if (gap == 0) { - zfs_panic_recover("zfs: rt=%s: adding segment " - "(offset=%llx size=%llx) overlapping with existing " - "one (offset=%llx size=%llx)", - ZFS_RT_NAME(rt), - (longlong_t)start, (longlong_t)size, - (longlong_t)rstart, (longlong_t)(rend - rstart)); - return; + if (panic_on_error) { + zfs_panic_recover("zfs: rt=%s: adding segment " + "(offset=%llx size=%llx) overlapping with " + "existing one (offset=%llx size=%llx)", + ZFS_RT_NAME(rt), + (longlong_t)start, (longlong_t)size, + (longlong_t)rstart, + (longlong_t)(rend - rstart)); + } + return (B_FALSE); } if (rstart <= start && rend >= end) { zfs_range_tree_adjust_fill(rt, rs, fill); - return; + return (B_TRUE); } if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL) @@ -381,8 +385,8 @@ zfs_range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill) size = end - start; zfs_btree_remove(&rt->rt_root, rs); - zfs_range_tree_add_impl(rt, start, size, fill); - return; + return (zfs_range_tree_add_impl(rt, start, size, fill, + panic_on_error)); } ASSERT0P(rs); @@ -472,17 +476,29 @@ zfs_range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill) zfs_range_tree_stat_incr(rt, rs); rt->rt_space += size + bridge_size; + return (B_TRUE); } void zfs_range_tree_add(void *arg, uint64_t start, uint64_t size) { - zfs_range_tree_add_impl(arg, start, size, size); + (void) zfs_range_tree_add_impl(arg, start, size, size, B_TRUE); } -static void +boolean_t +zfs_range_tree_try_add(zfs_range_tree_t *rt, uint64_t start, uint64_t size) +{ + /* + * Gap trees support overlapping additions by adjusting their fill, + * so only gap-free trees have a meaningful non-overlapping add. + */ + ASSERT0(rt->rt_gap); + return (zfs_range_tree_add_impl(rt, start, size, size, B_FALSE)); +} + +static boolean_t zfs_range_tree_remove_impl(zfs_range_tree_t *rt, uint64_t start, uint64_t size, - boolean_t do_fill) + boolean_t do_fill, boolean_t panic_on_error) { zfs_btree_index_t where; zfs_range_seg_t *rs; @@ -492,7 +508,11 @@ zfs_range_tree_remove_impl(zfs_range_tree_t *rt, uint64_t start, uint64_t size, boolean_t left_over, right_over; VERIFY3U(size, !=, 0); - VERIFY3U(size, <=, rt->rt_space); + if (size > rt->rt_space) { + if (panic_on_error) + VERIFY3U(size, <=, rt->rt_space); + return (B_FALSE); + } if (rt->rt_type == ZFS_RANGE_SEG64) ASSERT3U(start + size, >, start); @@ -502,10 +522,13 @@ zfs_range_tree_remove_impl(zfs_range_tree_t *rt, uint64_t start, uint64_t size, /* Make sure we completely overlap with someone */ if (rs == NULL) { - zfs_panic_recover("zfs: rt=%s: removing nonexistent segment " - "from range tree (offset=%llx size=%llx)", - ZFS_RT_NAME(rt), (longlong_t)start, (longlong_t)size); - return; + if (panic_on_error) { + zfs_panic_recover("zfs: rt=%s: removing nonexistent " + "segment from range tree (offset=%llx size=%llx)", + ZFS_RT_NAME(rt), (longlong_t)start, + (longlong_t)size); + } + return (B_FALSE); } rstart = zfs_rs_get_start(rs, rt); @@ -525,27 +548,33 @@ zfs_range_tree_remove_impl(zfs_range_tree_t *rt, uint64_t start, uint64_t size, size = end - start; } else { zfs_range_tree_adjust_fill(rt, rs, -size); - return; + return (B_TRUE); } } else if (rstart != start || rend != end) { - zfs_panic_recover("zfs: rt=%s: freeing partial segment " - "of gap tree (offset=%llx size=%llx) of " - "(offset=%llx size=%llx)", - ZFS_RT_NAME(rt), - (longlong_t)start, (longlong_t)size, - (longlong_t)rstart, (longlong_t)(rend - rstart)); - return; + if (panic_on_error) { + zfs_panic_recover("zfs: rt=%s: freeing partial " + "segment of gap tree (offset=%llx " + "size=%llx) of (offset=%llx size=%llx)", + ZFS_RT_NAME(rt), + (longlong_t)start, (longlong_t)size, + (longlong_t)rstart, + (longlong_t)(rend - rstart)); + } + return (B_FALSE); } } if (!(rstart <= start && rend >= end)) { - zfs_panic_recover("zfs: rt=%s: removing segment " - "(offset=%llx size=%llx) not completely overlapped by " - "existing one (offset=%llx size=%llx)", - ZFS_RT_NAME(rt), - (longlong_t)start, (longlong_t)size, - (longlong_t)rstart, (longlong_t)(rend - rstart)); - return; + if (panic_on_error) { + zfs_panic_recover("zfs: rt=%s: removing segment " + "(offset=%llx size=%llx) not completely " + "overlapped by existing one (offset=%llx " + "size=%llx)", ZFS_RT_NAME(rt), + (longlong_t)start, (longlong_t)size, + (longlong_t)rstart, + (longlong_t)(rend - rstart)); + } + return (B_FALSE); } left_over = (rstart != start); @@ -602,18 +631,30 @@ zfs_range_tree_remove_impl(zfs_range_tree_t *rt, uint64_t start, uint64_t size, } rt->rt_space -= size; + return (B_TRUE); } void zfs_range_tree_remove(void *arg, uint64_t start, uint64_t size) { - zfs_range_tree_remove_impl(arg, start, size, B_FALSE); + (void) zfs_range_tree_remove_impl(arg, start, size, B_FALSE, B_TRUE); +} + +boolean_t +zfs_range_tree_try_remove(zfs_range_tree_t *rt, uint64_t start, uint64_t size) +{ + /* + * Return without modifying the tree unless one segment completely + * contains the requested range. + */ + ASSERT0(rt->rt_gap); + return (zfs_range_tree_remove_impl(rt, start, size, B_FALSE, B_FALSE)); } void zfs_range_tree_remove_fill(zfs_range_tree_t *rt, uint64_t start, uint64_t size) { - zfs_range_tree_remove_impl(rt, start, size, B_TRUE); + (void) zfs_range_tree_remove_impl(rt, start, size, B_TRUE, B_TRUE); } void diff --git a/module/zfs/spa_log_spacemap.c b/module/zfs/spa_log_spacemap.c index 656f15b243a4..cf754b27512d 100644 --- a/module/zfs/spa_log_spacemap.c +++ b/module/zfs/spa_log_spacemap.c @@ -1437,14 +1437,19 @@ spa_ld_log_sm_data(spa_t *spa) for (metaslab_t *m = avl_first(&spa->spa_metaslabs_by_flushed); m != NULL; m = AVL_NEXT(&spa->spa_metaslabs_by_flushed, m)) { mutex_enter(&m->ms_lock); - m->ms_allocated_space = space_map_allocated(m->ms_sm) + - zfs_range_tree_space(m->ms_unflushed_allocs) - - zfs_range_tree_space(m->ms_unflushed_frees); - - metaslab_space_update(m->ms_group, - zfs_range_tree_space(m->ms_unflushed_allocs), 0, 0); - metaslab_space_update(m->ms_group, - -zfs_range_tree_space(m->ms_unflushed_frees), 0, 0); + if (m->ms_load_state != METASLAB_LOAD_UNLOADABLE && + !m->ms_smp_alloc_invalid) { + m->ms_allocated_space = space_map_allocated(m->ms_sm) + + zfs_range_tree_space(m->ms_unflushed_allocs) - + zfs_range_tree_space(m->ms_unflushed_frees); + + metaslab_space_update(m->ms_group, + zfs_range_tree_space(m->ms_unflushed_allocs), + 0, 0); + metaslab_space_update(m->ms_group, + -zfs_range_tree_space(m->ms_unflushed_frees), + 0, 0); + } ASSERT0(m->ms_weight & METASLAB_ACTIVE_MASK); metaslab_recalculate_weight_and_sort(m); @@ -1453,8 +1458,8 @@ spa_ld_log_sm_data(spa_t *spa) metaslab_unflushed_changes_memused(m); if (metaslab_debug_load && m->ms_sm != NULL) { - VERIFY0(metaslab_load(m)); - metaslab_set_selected_txg(m, 0); + if (metaslab_load(m) == 0) + metaslab_set_selected_txg(m, 0); } mutex_exit(&m->ms_lock); } diff --git a/module/zfs/space_map.c b/module/zfs/space_map.c index 13c606e9ff34..746c730aa9ac 100644 --- a/module/zfs/space_map.c +++ b/module/zfs/space_map.c @@ -82,9 +82,11 @@ sm_entry_is_double_word(uint64_t e) /* * Iterate through the space map, invoking the callback on each (non-debug) * space map entry. Stop after reading 'end' bytes of the space map. + * When repairing, return ECKSUM instead of asserting on malformed entries. */ -int -space_map_iterate(space_map_t *sm, uint64_t end, sm_cb_t callback, void *arg) +static int +space_map_iterate_impl(space_map_t *sm, uint64_t end, sm_cb_t callback, + void *arg, boolean_t repair) { uint64_t blksz = sm->sm_blksz; @@ -156,13 +158,24 @@ space_map_iterate(space_map_t *sm, uint64_t end, sm_cb_t callback, void *arg) /* move on to the second word */ block_cursor++; + if (repair && block_cursor >= block_end) { + error = SET_ERROR(ECKSUM); + break; + } + VERIFY3P(block_cursor, <, block_end); e = *block_cursor; - VERIFY3P(block_cursor, <=, block_end); type = SM2_TYPE_DECODE(e); raw_offset = SM2_OFFSET_DECODE(e); } + uint64_t map_units = sm->sm_size >> sm->sm_shift; + if (repair && (raw_offset >= map_units || + raw_run > map_units - raw_offset)) { + error = SET_ERROR(ECKSUM); + break; + } + uint64_t entry_offset = (raw_offset << sm->sm_shift) + sm->sm_start; uint64_t entry_run = raw_run << sm->sm_shift; @@ -190,6 +203,12 @@ space_map_iterate(space_map_t *sm, uint64_t end, sm_cb_t callback, void *arg) return (error); } +int +space_map_iterate(space_map_t *sm, uint64_t end, sm_cb_t callback, void *arg) +{ + return (space_map_iterate_impl(sm, end, callback, arg, B_FALSE)); +} + /* * Reads the entries from the last block of the space map into * buf in reverse order. Populates nwords with number of words @@ -394,13 +413,34 @@ space_map_incremental_destroy(space_map_t *sm, sm_cb_t callback, void *arg, typedef struct space_map_load_arg { space_map_t *smla_sm; zfs_range_tree_t *smla_rt; + zfs_range_tree_t *smla_repair_rt; maptype_t smla_type; + space_map_load_result_t *smla_result; } space_map_load_arg_t; +static void +space_map_load_record_repair(space_map_load_arg_t *smla, + const space_map_entry_t *sme, uint64_t free_bytes) +{ + space_map_load_result_t *result = smla->smla_result; + + ASSERT3P(result, !=, NULL); + ASSERT3U(free_bytes, <=, sme->sme_run); + + if (result->smlr_repaired_entries == 0) { + result->smlr_first_entry = *sme; + result->smlr_first_free_bytes = free_bytes; + } + result->smlr_last_entry = *sme; + result->smlr_last_free_bytes = free_bytes; + result->smlr_repaired_entries++; +} + static int space_map_load_callback(space_map_entry_t *sme, void *arg) { space_map_load_arg_t *smla = arg; + if (sme->sme_type == smla->smla_type) { VERIFY3U(zfs_range_tree_space(smla->smla_rt) + sme->sme_run, <=, smla->smla_sm->sm_size); @@ -414,33 +454,126 @@ space_map_load_callback(space_map_entry_t *sme, void *arg) return (0); } -/* - * Load the spacemap into the rangetree, like space_map_load. But only - * read the first 'length' bytes of the spacemap. - */ -int -space_map_load_length(space_map_t *sm, zfs_range_tree_t *rt, maptype_t maptype, - uint64_t length) +static int +space_map_load_repair_callback(space_map_entry_t *sme, void *arg) { - space_map_load_arg_t smla; + space_map_load_arg_t *smla = arg; + zfs_range_tree_t *rt = smla->smla_rt; + zfs_range_tree_t *repair_rt = smla->smla_repair_rt; + uint64_t overlap_start, overlap_size; + + ASSERT3P(smla->smla_result, !=, NULL); + ASSERT3S(smla->smla_type, ==, SM_FREE); + + boolean_t repair_overlap = repair_rt != NULL && + zfs_range_tree_find_in(repair_rt, sme->sme_offset, sme->sme_run, + &overlap_start, &overlap_size); + + if (!repair_overlap) { + if (sme->sme_type == SM_FREE) { + if (zfs_range_tree_try_add(rt, sme->sme_offset, + sme->sme_run)) + return (0); + } else { + if (zfs_range_tree_try_remove(rt, sme->sme_offset, + sme->sme_run)) + return (0); + } + } + + uint64_t free_before = zfs_range_tree_space(rt); + zfs_range_tree_clear(rt, sme->sme_offset, sme->sme_run); + free_before -= zfs_range_tree_space(rt); + /* + * An overlapping operation does not prove that any part of its range + * is unreferenced. Keep the complete range allocated for the rest of + * the replay and extend that quarantine through later overlapping + * entries. Condensation will persist this conservative state. + */ + if (repair_rt == NULL) { + repair_rt = zfs_range_tree_create_flags(NULL, rt->rt_type, NULL, + rt->rt_start, rt->rt_shift, 0, "space_map_load_repair"); + smla->smla_repair_rt = repair_rt; + } + zfs_range_tree_clear(repair_rt, sme->sme_offset, sme->sme_run); + zfs_range_tree_add(repair_rt, sme->sme_offset, sme->sme_run); + + space_map_load_record_repair(smla, sme, free_before); + return (0); +} + +static int +space_map_load_length_impl(space_map_t *sm, zfs_range_tree_t *rt, + maptype_t maptype, + uint64_t length, space_map_load_result_t *result) +{ + space_map_load_arg_t smla = { + .smla_sm = sm, + .smla_rt = rt, + .smla_type = maptype, + .smla_result = result, + }; VERIFY0(zfs_range_tree_space(rt)); if (maptype == SM_FREE) zfs_range_tree_add(rt, sm->sm_start, sm->sm_size); - smla.smla_rt = rt; - smla.smla_sm = sm; - smla.smla_type = maptype; - int err = space_map_iterate(sm, length, - space_map_load_callback, &smla); + if (result != NULL) { + memset(result, 0, sizeof (*result)); + } + + sm_cb_t callback = result == NULL ? space_map_load_callback : + space_map_load_repair_callback; + int err = space_map_iterate_impl(sm, length, callback, &smla, + result != NULL); + + if (result != NULL && err == 0) { + result->smlr_allocated = + sm->sm_size - zfs_range_tree_space(rt); + if (smla.smla_repair_rt != NULL) { + result->smlr_affected_bytes = + zfs_range_tree_space(smla.smla_repair_rt); + } + } if (err != 0) zfs_range_tree_vacate(rt, NULL, NULL); + if (smla.smla_repair_rt != NULL) { + zfs_range_tree_vacate(smla.smla_repair_rt, NULL, NULL); + zfs_range_tree_destroy(smla.smla_repair_rt); + } + return (err); } +/* + * Load the spacemap into the rangetree, like space_map_load. But only + * read the first 'length' bytes of the spacemap. + */ +int +space_map_load_length(space_map_t *sm, zfs_range_tree_t *rt, maptype_t maptype, + uint64_t length) +{ + return (space_map_load_length_impl(sm, rt, maptype, length, NULL)); +} + +/* + * Load a free-space map while conservatively quarantining entries that + * overlap the state reconstructed so far. The caller must persist the + * resulting allocation state before making the range tree allocatable. + */ +int +space_map_load_length_repair(space_map_t *sm, zfs_range_tree_t *rt, + uint64_t length, space_map_load_result_t *result) +{ + if (result == NULL) + return (SET_ERROR(EINVAL)); + + return (space_map_load_length_impl(sm, rt, SM_FREE, length, result)); +} + /* * Load the space map disk into the specified range tree. Segments of maptype * are added to the range tree, other segment types are removed. diff --git a/module/zfs/vdev_initialize.c b/module/zfs/vdev_initialize.c index b7709317df5e..5c431cd0ca64 100644 --- a/module/zfs/vdev_initialize.c +++ b/module/zfs/vdev_initialize.c @@ -448,7 +448,10 @@ vdev_initialize_calculate_progress(vdev_t *vd) * metaslab. Load it and walk the free tree for more accurate * progress estimation. */ - VERIFY0(metaslab_load(msp)); + if (metaslab_load(msp) != 0) { + mutex_exit(&msp->ms_lock); + continue; + } zfs_btree_index_t where; zfs_range_tree_t *rt = msp->ms_allocatable; @@ -574,7 +577,16 @@ vdev_initialize_thread(void *arg) mutex_enter(&msp->ms_lock); if (!msp->ms_loaded && !msp->ms_loading) unload_when_done = B_TRUE; - VERIFY0(metaslab_load(msp)); + error = metaslab_load(msp); + if (error != 0) { + mutex_exit(&msp->ms_lock); + metaslab_enable(msp, B_FALSE, unload_when_done); + spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); + zfs_dbgmsg("initialize: unable to load metaslab %llu " + "on vdev %s: error %d", + (u_longlong_t)msp->ms_id, vd->vdev_path, error); + break; + } zfs_range_tree_walk(msp->ms_allocatable, vdev_initialize_range_add, vd); @@ -603,7 +615,10 @@ vdev_initialize_thread(void *arg) mutex_enter(&vd->vdev_initialize_lock); if (!vd->vdev_initialize_exit_wanted) { - if (vdev_writeable(vd)) { + if (error != 0 && vdev_writeable(vd)) { + vdev_initialize_change_state(vd, + VDEV_INITIALIZE_SUSPENDED); + } else if (vdev_writeable(vd)) { vdev_initialize_change_state(vd, VDEV_INITIALIZE_COMPLETE); } else if (vd->vdev_faulted) { diff --git a/module/zfs/vdev_raidz.c b/module/zfs/vdev_raidz.c index 5fb568536691..96e235a93295 100644 --- a/module/zfs/vdev_raidz.c +++ b/module/zfs/vdev_raidz.c @@ -5009,7 +5009,7 @@ spa_raidz_expand_thread(void *arg, zthr_t *zthr) continue; } - VERIFY0(metaslab_load(msp)); + boolean_t loaded = (metaslab_load(msp) == 0); /* * We want to copy everything except the free (allocatable) @@ -5024,8 +5024,10 @@ spa_raidz_expand_thread(void *arg, zthr_t *zthr) metaslab_rt_name(msp->ms_group, msp, "spa_raidz_expand_thread:rt")); zfs_range_tree_add(rt, msp->ms_start, msp->ms_size); - zfs_range_tree_walk(msp->ms_allocatable, zfs_range_tree_remove, - rt); + if (loaded) { + zfs_range_tree_walk(msp->ms_allocatable, + zfs_range_tree_remove, rt); + } mutex_exit(&msp->ms_lock); /* diff --git a/module/zfs/vdev_trim.c b/module/zfs/vdev_trim.c index a97f6650a81c..3cfb0fefd764 100644 --- a/module/zfs/vdev_trim.c +++ b/module/zfs/vdev_trim.c @@ -728,7 +728,10 @@ vdev_trim_calculate_progress(vdev_t *vd) * metaslab. Load it and walk the free tree for more * accurate progress estimation. */ - VERIFY0(metaslab_load(msp)); + if (metaslab_load(msp) != 0) { + mutex_exit(&msp->ms_lock); + continue; + } zfs_range_tree_t *rt = msp->ms_allocatable; zfs_btree_t *bt = &rt->rt_root; @@ -857,10 +860,11 @@ vdev_trim_range_add(void *arg, uint64_t start, uint64_t size) */ if (zfs_flags & ZFS_DEBUG_TRIM) { metaslab_t *msp = ta->trim_msp; - VERIFY0(metaslab_load(msp)); + if (metaslab_load(msp) != 0) + return; VERIFY3B(msp->ms_loaded, ==, B_TRUE); - VERIFY(zfs_range_tree_contains(msp->ms_allocatable, start, - size)); + VERIFY(zfs_range_tree_contains(msp->ms_allocatable, + start, size)); } ASSERT(vd->vdev_ops->vdev_op_leaf); @@ -935,7 +939,16 @@ vdev_trim_thread(void *arg) spa_config_exit(spa, SCL_CONFIG, FTAG); metaslab_disable(msp); mutex_enter(&msp->ms_lock); - VERIFY0(metaslab_load(msp)); + error = metaslab_load(msp); + if (error != 0) { + mutex_exit(&msp->ms_lock); + metaslab_enable(msp, B_FALSE, B_FALSE); + spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); + zfs_dbgmsg("trim: unable to load metaslab %llu on " + "vdev %s: error %d", + (u_longlong_t)msp->ms_id, vd->vdev_path, error); + break; + } /* * If a partial TRIM was requested skip metaslabs which have @@ -970,7 +983,11 @@ vdev_trim_thread(void *arg) mutex_enter(&vd->vdev_trim_lock); if (!vd->vdev_trim_exit_wanted) { - if (vdev_writeable(vd)) { + if (error != 0 && vdev_writeable(vd)) { + vdev_trim_change_state(vd, VDEV_TRIM_SUSPENDED, + vd->vdev_trim_rate, vd->vdev_trim_partial, + vd->vdev_trim_secure); + } else if (vdev_writeable(vd)) { vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE, vd->vdev_trim_rate, vd->vdev_trim_partial, vd->vdev_trim_secure); @@ -1280,7 +1297,8 @@ vdev_autotrim_thread(void *arg) * Skip the metaslab when it has never been allocated * or when there are no recent frees to trim. */ - if (msp->ms_sm == NULL || + if (msp->ms_load_state == METASLAB_LOAD_UNLOADABLE || + msp->ms_sm == NULL || zfs_range_tree_is_empty(msp->ms_trim)) { mutex_exit(&msp->ms_lock); metaslab_enable(msp, B_FALSE, B_FALSE); @@ -1413,10 +1431,11 @@ vdev_autotrim_thread(void *arg) */ if (zfs_flags & ZFS_DEBUG_TRIM) { mutex_enter(&msp->ms_lock); - VERIFY0(metaslab_load(msp)); - VERIFY3P(tap[0].trim_msp, ==, msp); - zfs_range_tree_walk(trim_tree, - vdev_trim_range_verify, &tap[0]); + if (metaslab_load(msp) == 0) { + VERIFY3P(tap[0].trim_msp, ==, msp); + zfs_range_tree_walk(trim_tree, + vdev_trim_range_verify, &tap[0]); + } mutex_exit(&msp->ms_lock); } diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index 96e7f605229d..7d2321cc5220 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -392,7 +392,8 @@ tags = ['functional', 'cli_root', 'zfs_wait'] [tests/functional/cli_root/zhack] tests = ['zhack_label_repair_001', 'zhack_label_repair_002', - 'zhack_label_repair_003', 'zhack_label_repair_004', 'zhack_metaslab_leak'] + 'zhack_label_repair_003', 'zhack_label_repair_004', 'zhack_metaslab_leak', + 'zhack_metaslab_repair'] pre = post = tags = ['functional', 'cli_root', 'zhack'] diff --git a/tests/unit/.gitignore b/tests/unit/.gitignore index 3f240887d5cc..470234ffe934 100644 --- a/tests/unit/.gitignore +++ b/tests/unit/.gitignore @@ -5,4 +5,5 @@ /test_zap /test_namecheck /test_btree +/test_range_tree /test_sha2 diff --git a/tests/unit/Makefile.am b/tests/unit/Makefile.am index e1a44dcb295e..dc2508fbdb47 100644 --- a/tests/unit/Makefile.am +++ b/tests/unit/Makefile.am @@ -18,6 +18,7 @@ UNIT_TESTS = \ %D%/test_zap \ %D%/test_namecheck \ %D%/test_btree \ + %D%/test_range_tree \ %D%/test_fletcher \ %D%/test_sha2 noinst_PROGRAMS = $(UNIT_TESTS) @@ -69,6 +70,20 @@ nodist_%C%_test_btree_SOURCES = \ libunit.la +%C%_test_range_tree_CFLAGS = $(AM_CFLAGS) + +nodist_%C%_test_range_tree_SOURCES = \ + module/zfs/btree.c \ + module/zfs/range_tree.c + +%C%_test_range_tree_SOURCES = \ + %D%/test_range_tree.c + +%C%_test_range_tree_LDADD = \ + libspl.la \ + libunit.la + + %C%_test_fletcher_CFLAGS = $(AM_CFLAGS) nodist_%C%_test_fletcher_SOURCES = \ diff --git a/tests/unit/test_range_tree.c b/tests/unit/test_range_tree.c new file mode 100644 index 000000000000..aae9011edfac --- /dev/null +++ b/tests/unit/test_range_tree.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + */ + +/* + * Copyright (c) 2026, James Hilliard. + */ + +#include +#include + +#include "unit.h" + +static zfs_range_tree_t * +range_tree_create(void) +{ + return (zfs_range_tree_create(NULL, ZFS_RANGE_SEG64, NULL, 0, 0)); +} + +static void +range_tree_destroy(zfs_range_tree_t *rt) +{ + zfs_range_tree_vacate(rt, NULL, NULL); + zfs_range_tree_destroy(rt); +} + +static MunitResult +test_try_add(const MunitParameter params[], void *data) +{ + (void) params, (void) data; + + zfs_range_tree_t *rt = range_tree_create(); + + unit_true(zfs_range_tree_try_add(rt, 0x1000, 0x1000)); + unit_true(zfs_range_tree_try_add(rt, 0x3000, 0x1000)); + unit_true(zfs_range_tree_try_add(rt, 0x2000, 0x1000)); + unit_eq(zfs_range_tree_numsegs(rt), 1); + unit_eq(zfs_range_tree_space(rt), 0x3000); + unit_true(zfs_range_tree_contains(rt, 0x1000, 0x3000)); + + range_tree_destroy(rt); + return (MUNIT_OK); +} + +static MunitResult +test_try_add_overlap(const MunitParameter params[], void *data) +{ + (void) params, (void) data; + + zfs_range_tree_t *rt = range_tree_create(); + + zfs_range_tree_add(rt, 0x1000, 0x2000); + zfs_range_tree_add(rt, 0x5000, 0x1000); + + unit_false(zfs_range_tree_try_add(rt, 0x1000, 0x1000)); + unit_false(zfs_range_tree_try_add(rt, 0x2800, 0x1000)); + unit_eq(zfs_range_tree_numsegs(rt), 2); + unit_eq(zfs_range_tree_space(rt), 0x3000); + unit_true(zfs_range_tree_contains(rt, 0x1000, 0x2000)); + unit_true(zfs_range_tree_contains(rt, 0x5000, 0x1000)); + unit_false(zfs_range_tree_contains(rt, 0x3000, 0x2000)); + + range_tree_destroy(rt); + return (MUNIT_OK); +} + +static MunitResult +test_try_remove(const MunitParameter params[], void *data) +{ + (void) params, (void) data; + + zfs_range_tree_t *rt = range_tree_create(); + + zfs_range_tree_add(rt, 0x1000, 0x3000); + unit_true(zfs_range_tree_try_remove(rt, 0x2000, 0x1000)); + unit_eq(zfs_range_tree_numsegs(rt), 2); + unit_eq(zfs_range_tree_space(rt), 0x2000); + unit_true(zfs_range_tree_contains(rt, 0x1000, 0x1000)); + unit_true(zfs_range_tree_contains(rt, 0x3000, 0x1000)); + unit_false(zfs_range_tree_contains(rt, 0x2000, 0x1000)); + + unit_true(zfs_range_tree_try_remove(rt, 0x1000, 0x1000)); + unit_true(zfs_range_tree_try_remove(rt, 0x3000, 0x1000)); + unit_true(zfs_range_tree_is_empty(rt)); + + range_tree_destroy(rt); + return (MUNIT_OK); +} + +static MunitResult +test_try_remove_missing(const MunitParameter params[], void *data) +{ + (void) params, (void) data; + + zfs_range_tree_t *rt = range_tree_create(); + + zfs_range_tree_add(rt, 0x1000, 0x2000); + zfs_range_tree_add(rt, 0x5000, 0x1000); + + unit_false(zfs_range_tree_try_remove(rt, 0x3000, 0x1000)); + unit_false(zfs_range_tree_try_remove(rt, 0x2800, 0x3000)); + unit_false(zfs_range_tree_try_remove(rt, 0x1000, 0x4000)); + unit_eq(zfs_range_tree_numsegs(rt), 2); + unit_eq(zfs_range_tree_space(rt), 0x3000); + unit_true(zfs_range_tree_contains(rt, 0x1000, 0x2000)); + unit_true(zfs_range_tree_contains(rt, 0x5000, 0x1000)); + unit_false(zfs_range_tree_contains(rt, 0x3000, 0x2000)); + + range_tree_destroy(rt); + return (MUNIT_OK); +} + +static const MunitTest range_tree_tests[] = { + UNIT_TEST("try_add", test_try_add), + UNIT_TEST("try_add_overlap", test_try_add_overlap), + UNIT_TEST("try_remove", test_try_remove), + UNIT_TEST("try_remove_missing", test_try_remove_missing), + { 0 }, +}; + +static const MunitSuite range_tree_test_suite = { + "range_tree.", + range_tree_tests, + NULL, + 1, + MUNIT_SUITE_OPTION_NONE, +}; + +int +main(int argc, char **argv) +{ + int ret; + + zfs_btree_init(); + ret = munit_suite_main(&range_tree_test_suite, NULL, argc, argv); + zfs_btree_fini(); + return (ret); +} diff --git a/tests/zfs-tests/tests/Makefile.am b/tests/zfs-tests/tests/Makefile.am index c7bae4cb7ab0..d4453757d9f0 100644 --- a/tests/zfs-tests/tests/Makefile.am +++ b/tests/zfs-tests/tests/Makefile.am @@ -1084,6 +1084,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/cli_root/zhack/zhack_label_repair_003.ksh \ functional/cli_root/zhack/zhack_label_repair_004.ksh \ functional/cli_root/zhack/zhack_metaslab_leak.ksh \ + functional/cli_root/zhack/zhack_metaslab_repair.ksh \ functional/cli_root/zpool_add/add_nested_replacing_spare.ksh \ functional/cli_root/zpool_add/add-o_ashift.ksh \ functional/cli_root/zpool_add/add_prop_ashift.ksh \ diff --git a/tests/zfs-tests/tests/functional/cli_root/zhack/zhack_metaslab_repair.ksh b/tests/zfs-tests/tests/functional/cli_root/zhack/zhack_metaslab_repair.ksh new file mode 100755 index 000000000000..fc47b1cf59cc --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zhack/zhack_metaslab_repair.ksh @@ -0,0 +1,185 @@ +#!/bin/ksh +# SPDX-License-Identifier: CDDL-1.0 +# ZTS supplies STF_SUITE, TESTPOOL, and DISKS. +# shellcheck disable=SC2154 + +# +# Verify that repairable metaslab damage is conservatively replaced with a +# canonical space map and malformed maps remain unavailable. +# + +. "$STF_SUITE"/include/libtest.shlib + +verify_runnable "global" + +typeset old_debug_load +typeset old_keep_logs +typeset corruption +typeset tmpdir + +function cleanup +{ + log_must set_tunable64 METASLAB_DEBUG_LOAD "$old_debug_load" + log_must set_tunable64 KEEP_LOG_SPACEMAPS_AT_EXPORT "$old_keep_logs" + if poolexists "$TESTPOOL"; then + destroy_pool "$TESTPOOL" + fi + rm -rf "$tmpdir" +} + +function wait_for_repair +{ + typeset pool=$1 + typeset -i attempt + + for ((attempt = 0; attempt < 10; attempt++)); do + if zpool history -il "$pool" | grep -q "metaslab repair"; then + return 0 + fi + zpool sync "$pool" || return 1 + done + + zpool history -il "$pool" | grep -q "metaslab repair" +} + +function run_repair_test +{ + typeset corruption=$1 + typeset log_spacemap=$2 + typeset checksum + typeset repaired_checksum + typeset injection + typeset injection_log="$tmpdir/injection-$corruption-$log_spacemap" + typeset allocated_map="$tmpdir/map-$corruption-$log_spacemap" + typeset vdev + typeset offset + typeset size + typeset -a vdevs + + read -r -A vdevs <<< "$DISKS" + + log_note "testing $corruption corruption with log_spacemap " \ + "$log_spacemap" + log_must zpool create -o feature@log_spacemap="$log_spacemap" \ + "$TESTPOOL" "${vdevs[@]}" + log_must dd if=/dev/urandom of="/$TESTPOOL/data" bs=1M count=16 + checksum=$(cksum "/$TESTPOOL/data") + log_must zpool sync "$TESTPOOL" + log_must dd if=/dev/urandom of="/$TESTPOOL/churn" bs=128k count=10 + log_must zpool sync "$TESTPOOL" + + if [[ "$log_spacemap" == "enabled" ]]; then + log_must set_tunable64 KEEP_LOG_SPACEMAPS_AT_EXPORT 1 + log_must_busy zpool export "$TESTPOOL" + log_must eval "zhack -o zfs_keep_log_spacemaps_at_export=1 " \ + "metaslab corrupt $TESTPOOL $corruption > $injection_log" + else + log_must set_tunable64 KEEP_LOG_SPACEMAPS_AT_EXPORT 0 + log_must_busy zpool export "$TESTPOOL" + log_must eval "zhack metaslab corrupt $TESTPOOL $corruption " \ + "> $injection_log" + fi + + # Invalid summaries are scheduled for repair during metaslab setup. + if [[ "$corruption" == "invalid-summary" ]]; then + log_must set_tunable64 METASLAB_DEBUG_LOAD 0 + else + # Load every metaslab so import finds the injected overlap. + log_must set_tunable64 METASLAB_DEBUG_LOAD 1 + fi + log_must zpool import "$TESTPOOL" + + log_must wait_for_repair "$TESTPOOL" + + log_must set_tunable64 METASLAB_DEBUG_LOAD 0 + log_must set_tunable64 KEEP_LOG_SPACEMAPS_AT_EXPORT 0 + log_must_busy zpool export "$TESTPOOL" + + # zdb -b enables leak tracking, which strictly loads every space map. + log_must zdb -e -b "$TESTPOOL" + log_must eval "zdb -e -m --allocated-map $TESTPOOL > $allocated_map" + + if [[ "$corruption" != "invalid-summary" ]]; then + injection=$(<"$injection_log") + vdev=${injection#*vdev=} + vdev=${vdev%% *} + offset=${injection#*offset=} + offset=${offset%% *} + size=${injection#*size=} + size=${size%% *} + + # The complete ambiguous range must remain allocated. + log_must awk -v target_vdev="$vdev" \ + -v target_start="$offset" -v target_size="$size" " + \$1 == \"vdev\" { current_vdev = \$2 } + current_vdev == target_vdev && \$1 == \"ALLOC:\" && + \$2 <= target_start && + \$2 + \$3 >= target_start + target_size { found = 1 } + END { exit !found } + " "$allocated_map" + fi + + log_must zpool import "$TESTPOOL" + repaired_checksum=$(cksum "/$TESTPOOL/data") + log_must test "$repaired_checksum" = "$checksum" + log_must zpool destroy "$TESTPOOL" +} + +function run_unloadable_test +{ + typeset checksum + typeset reopened_checksum + typeset before + typeset after + typeset -a vdevs + + read -r -A vdevs <<< "$DISKS" + + log_note "testing an invalid metaslab space map entry" + log_must zpool create -o feature@log_spacemap=disabled \ + "$TESTPOOL" "${vdevs[@]}" + log_must dd if=/dev/urandom of="/$TESTPOOL/data" bs=1M count=16 + checksum=$(cksum "/$TESTPOOL/data") + log_must zpool sync "$TESTPOOL" + log_must_busy zpool export "$TESTPOOL" + + before=$(kstat metaslab_stats.load_unloadable) + log_must zhack metaslab corrupt "$TESTPOOL" invalid-entry + log_must set_tunable64 METASLAB_DEBUG_LOAD 1 + log_must zpool import "$TESTPOOL" + + after=$(kstat metaslab_stats.load_unloadable) + log_must test "$after" -gt "$before" + + # Waiting succeeds after the invalid metaslab suspends initialization. + log_must zpool initialize -w "$TESTPOOL" "${vdevs[0]}" + log_must eval "zpool status -i $TESTPOOL | grep -F ${vdevs[0]} | " \ + "grep -q suspended" + + log_must zpool initialize -c "$TESTPOOL" "${vdevs[0]}" + log_must_busy zpool export "$TESTPOOL" + log_must zpool import "$TESTPOOL" + + log_must set_tunable64 METASLAB_DEBUG_LOAD 0 + reopened_checksum=$(cksum "/$TESTPOOL/data") + log_must test "$reopened_checksum" = "$checksum" + log_must zpool destroy "$TESTPOOL" +} + +log_assert "damaged metaslab space maps are repaired or quarantined safely" + +old_debug_load=$(get_tunable METASLAB_DEBUG_LOAD) +old_keep_logs=$(get_tunable KEEP_LOG_SPACEMAPS_AT_EXPORT) +tmpdir=$(mktemp -d "$TEST_BASE_DIR/zhack_metaslab_repair.XXXXXX") || + log_fail "failed to create temporary directory" +log_onexit cleanup + +for corruption in duplicate-free partial-free duplicate-alloc partial-alloc \ + chained invalid-summary; do + run_repair_test "$corruption" enabled +done + +run_repair_test partial-free disabled +run_unloadable_test + +log_pass "metaslab damage was handled without changing referenced data"