diff --git a/cmd/zhack.c b/cmd/zhack.c index e9306e63f4c3..96087f9a8b34 100644 --- a/cmd/zhack.c +++ b/cmd/zhack.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -112,7 +113,11 @@ usage(void) " : path to vdev\n" "\n" " metaslab leak \n" - " apply allocation map from zdb to specified pool\n"); + " apply allocation map from zdb to specified pool\n" + " metaslab corrupt \n" + " inject duplicate-free, partial-free, duplicate-alloc,\n" + " partial-alloc, chained, invalid-summary, or\n" + " invalid-entry damage\n"); exit(1); } @@ -785,6 +790,241 @@ zhack_do_metaslab_leak(int argc, char **argv) spa_close(spa, FTAG); } +typedef enum zhack_metaslab_corruption { + ZHACK_METASLAB_DUPLICATE_FREE, + ZHACK_METASLAB_PARTIAL_FREE, + ZHACK_METASLAB_DUPLICATE_ALLOC, + ZHACK_METASLAB_PARTIAL_ALLOC, + ZHACK_METASLAB_CHAINED, + ZHACK_METASLAB_INVALID_SUMMARY, + ZHACK_METASLAB_INVALID_ENTRY +} zhack_metaslab_corruption_t; + +typedef struct zhack_metaslab_corrupt_arg { + zhack_metaslab_corruption_t zmca_type; + const char *zmca_name; + boolean_t zmca_injected; + uint64_t zmca_vdev; + uint64_t zmca_metaslab; + uint64_t zmca_offset; + uint64_t zmca_size; +} zhack_metaslab_corrupt_arg_t; + +static boolean_t +zhack_metaslab_corrupt_range(metaslab_t *msp, + zhack_metaslab_corruption_t type, uint64_t *offset, uint64_t *size) +{ + zfs_range_tree_t *rt = msp->ms_allocatable; + uint64_t unit = 1ULL << msp->ms_sm->sm_shift; + uint64_t cursor = msp->ms_start; + zfs_btree_index_t where; + zfs_range_seg_t *rs; + + for (rs = zfs_btree_first(&rt->rt_root, &where); rs != NULL; + rs = zfs_btree_next(&rt->rt_root, &where, &where)) { + uint64_t start = zfs_rs_get_start(rs, rt); + uint64_t end = zfs_rs_get_end(rs, rt); + + if ((type == ZHACK_METASLAB_DUPLICATE_ALLOC) && + start - cursor >= unit) { + *offset = cursor; + *size = unit; + return (B_TRUE); + } + if (type == ZHACK_METASLAB_DUPLICATE_FREE && + end - start >= unit) { + *offset = start; + *size = unit; + return (B_TRUE); + } + if ((type == ZHACK_METASLAB_PARTIAL_FREE || + type == ZHACK_METASLAB_PARTIAL_ALLOC) && + start - msp->ms_start >= unit) { + *offset = start - unit; + *size = 2 * unit; + return (B_TRUE); + } + if (type == ZHACK_METASLAB_CHAINED && end - start >= 2 * unit) { + *offset = start; + *size = 2 * unit; + return (B_TRUE); + } + if ((type == ZHACK_METASLAB_PARTIAL_FREE || + type == ZHACK_METASLAB_PARTIAL_ALLOC) && + msp->ms_start + msp->ms_size - end >= unit) { + *offset = end - unit; + *size = 2 * unit; + return (B_TRUE); + } + cursor = end; + } + + if (type == ZHACK_METASLAB_DUPLICATE_ALLOC && + msp->ms_start + msp->ms_size - cursor >= unit) { + *offset = cursor; + *size = unit; + return (B_TRUE); + } + + return (B_FALSE); +} + +static void +zhack_metaslab_corrupt_invalid_entry(space_map_t *sm, dmu_tx_t *tx) +{ + uint64_t entry = SM_OFFSET_ENCODE(sm->sm_size >> sm->sm_shift) | + SM_TYPE_ENCODE(SM_ALLOC) | SM_RUN_ENCODE(1); + + dmu_buf_will_dirty(sm->sm_dbuf, tx); + dmu_write(sm->sm_os, space_map_object(sm), space_map_length(sm), + sizeof (entry), &entry, tx, DMU_READ_NO_PREFETCH); + sm->sm_phys->smp_length += sizeof (entry); +} + +static void +zhack_metaslab_corrupt_sync(void *arg, dmu_tx_t *tx) +{ + zhack_metaslab_corrupt_arg_t *zmca = arg; + spa_t *spa = dmu_tx_pool(tx)->dp_spa; + vdev_t *rvd = spa->spa_root_vdev; + + for (uint64_t c = 0; c < rvd->vdev_children; c++) { + vdev_t *vd = rvd->vdev_child[c]; + if (vd->vdev_mg == NULL) + continue; + + for (uint64_t m = 0; m < vd->vdev_ms_count; m++) { + metaslab_t *msp = vd->vdev_ms[m]; + + mutex_enter(&msp->ms_lock); + if (msp->ms_sm == NULL || metaslab_load(msp) != 0) { + mutex_exit(&msp->ms_lock); + continue; + } + + uint64_t offset = 0; + uint64_t size = 0; + boolean_t invalid_summary = zmca->zmca_type == + ZHACK_METASLAB_INVALID_SUMMARY; + boolean_t invalid_entry = zmca->zmca_type == + ZHACK_METASLAB_INVALID_ENTRY; + if (invalid_summary) { + dmu_buf_will_dirty(msp->ms_sm->sm_dbuf, tx); + msp->ms_sm->sm_phys->smp_alloc = + -(int64_t)(1ULL << msp->ms_sm->sm_shift); + } else if (invalid_entry) { + offset = msp->ms_start + msp->ms_size; + size = 1ULL << msp->ms_sm->sm_shift; + } else if (!zhack_metaslab_corrupt_range(msp, + zmca->zmca_type, &offset, &size)) { + mutex_exit(&msp->ms_lock); + continue; + } + + space_map_t *sm = msp->ms_sm; + zfs_range_tree_t *rt = NULL; + if (!invalid_summary && !invalid_entry) { + rt = zfs_range_tree_create(NULL, + ZFS_RANGE_SEG64, NULL, 0, 0); + uint64_t first_size = + zmca->zmca_type == ZHACK_METASLAB_CHAINED ? + size / 2 : size; + zfs_range_tree_add(rt, offset, first_size); + } + mutex_exit(&msp->ms_lock); + + if (invalid_entry) { + zhack_metaslab_corrupt_invalid_entry(sm, tx); + } else if (rt != NULL) { + maptype_t maptype = + zmca->zmca_type == + ZHACK_METASLAB_DUPLICATE_ALLOC || + zmca->zmca_type == + ZHACK_METASLAB_PARTIAL_ALLOC ? + SM_ALLOC : SM_FREE; + space_map_write(sm, rt, maptype, + SM_NO_VDEVID, tx); + if (zmca->zmca_type == ZHACK_METASLAB_CHAINED) { + zfs_range_tree_vacate(rt, NULL, NULL); + zfs_range_tree_add(rt, offset, size); + space_map_write(sm, rt, SM_ALLOC, + SM_NO_VDEVID, tx); + } + zfs_range_tree_vacate(rt, NULL, NULL); + zfs_range_tree_destroy(rt); + } + + zmca->zmca_injected = B_TRUE; + zmca->zmca_vdev = vd->vdev_id; + zmca->zmca_metaslab = msp->ms_id; + zmca->zmca_offset = offset; + zmca->zmca_size = size; + return; + } + } +} + +static void +zhack_do_metaslab_corrupt(int argc, char **argv) +{ + char *target; + spa_t *spa; + + argc--; + argv++; + if (argc != 2) { + (void) fprintf(stderr, + "error: expected a pool name and corruption type\n"); + usage(); + } + target = argv[0]; + + zhack_metaslab_corrupt_arg_t zmca = { + .zmca_name = argv[1], + }; + if (strcmp(zmca.zmca_name, "duplicate-free") == 0) { + zmca.zmca_type = ZHACK_METASLAB_DUPLICATE_FREE; + } else if (strcmp(zmca.zmca_name, "partial-free") == 0) { + zmca.zmca_type = ZHACK_METASLAB_PARTIAL_FREE; + } else if (strcmp(zmca.zmca_name, "duplicate-alloc") == 0) { + zmca.zmca_type = ZHACK_METASLAB_DUPLICATE_ALLOC; + } else if (strcmp(zmca.zmca_name, "partial-alloc") == 0) { + zmca.zmca_type = ZHACK_METASLAB_PARTIAL_ALLOC; + } else if (strcmp(zmca.zmca_name, "chained") == 0) { + zmca.zmca_type = ZHACK_METASLAB_CHAINED; + } else if (strcmp(zmca.zmca_name, "invalid-summary") == 0) { + zmca.zmca_type = ZHACK_METASLAB_INVALID_SUMMARY; + } else if (strcmp(zmca.zmca_name, "invalid-entry") == 0) { + zmca.zmca_type = ZHACK_METASLAB_INVALID_ENTRY; + } else { + (void) fprintf(stderr, "error: unknown corruption type: %s\n", + zmca.zmca_name); + usage(); + } + + /* + * Committing an intentionally inconsistent space map can make its + * allocation summary disagree with the loaded metaslab. Leave other + * debug checks enabled while allowing the injector to close the pool. + */ + zfs_flags &= ~ZFS_DEBUG_METASLAB_VERIFY; + zhack_spa_open(target, B_FALSE, FTAG, &spa); + + VERIFY0(dsl_sync_task(spa_name(spa), NULL, + zhack_metaslab_corrupt_sync, &zmca, 5, + ZFS_SPACE_CHECK_NONE)); + if (!zmca.zmca_injected) { + fatal(spa, FTAG, "no suitable metaslab range available in '%s'", + target); + } + + (void) printf("injected %s: vdev=%" PRIu64 + " metaslab=%" PRIu64 " offset=%" PRIu64 " size=%" PRIu64 "\n", + zmca.zmca_name, zmca.zmca_vdev, zmca.zmca_metaslab, + zmca.zmca_offset, zmca.zmca_size); + spa_close(spa, FTAG); +} + static int zhack_do_metaslab(int argc, char **argv) { @@ -801,6 +1041,8 @@ zhack_do_metaslab(int argc, char **argv) subcommand = argv[0]; if (strcmp(subcommand, "leak") == 0) { zhack_do_metaslab_leak(argc, argv); + } else if (strcmp(subcommand, "corrupt") == 0) { + zhack_do_metaslab_corrupt(argc, argv); } else { (void) fprintf(stderr, "error: unknown subcommand: %s\n", subcommand); diff --git a/include/sys/metaslab_impl.h b/include/sys/metaslab_impl.h index 78b56529a399..92dbeca75c39 100644 --- a/include/sys/metaslab_impl.h +++ b/include/sys/metaslab_impl.h @@ -338,6 +338,19 @@ struct metaslab_group { * ensure that allocations are not performed on the metaslab that is * being written. */ +typedef enum metaslab_load_state { + METASLAB_LOAD_NORMAL = 0, + METASLAB_LOAD_DEFERRED, + METASLAB_LOAD_UNLOADABLE, +} metaslab_load_state_t; + +typedef struct metaslab_repair { + int64_t msr_smp_alloc; + uint64_t msr_allocated; + uint64_t msr_entries; + uint64_t msr_bytes; +} metaslab_repair_t; + struct metaslab { /* * This is the main lock of the metaslab and its purpose is to @@ -404,6 +417,12 @@ struct metaslab { boolean_t ms_condensing; /* condensing? */ boolean_t ms_condense_wanted; + /* Details retained until a repaired space map is condensed. */ + metaslab_repair_t ms_repair; + /* The on-disk smp_alloc is outside the metaslab's valid range. */ + boolean_t ms_smp_alloc_invalid; + metaslab_load_state_t ms_load_state; + /* * The number of consumers which have disabled the metaslab. */ diff --git a/include/sys/range_tree.h b/include/sys/range_tree.h index 0f6def36f9f6..8c0fe129c48f 100644 --- a/include/sys/range_tree.h +++ b/include/sys/range_tree.h @@ -310,7 +310,17 @@ uint64_t zfs_range_tree_max(zfs_range_tree_t *rt); uint64_t zfs_range_tree_span(zfs_range_tree_t *rt); void zfs_range_tree_add(void *arg, uint64_t start, uint64_t size); +/* + * For gap-free trees, try_add() succeeds only when the requested range does + * not overlap the tree, and try_remove() succeeds only when one segment + * contains the entire range. Both return B_FALSE without modifying the tree + * otherwise. + */ +boolean_t zfs_range_tree_try_add(zfs_range_tree_t *rt, uint64_t start, + uint64_t size); void zfs_range_tree_remove(void *arg, uint64_t start, uint64_t size); +boolean_t zfs_range_tree_try_remove(zfs_range_tree_t *rt, uint64_t start, + uint64_t size); void zfs_range_tree_remove_fill(zfs_range_tree_t *rt, uint64_t start, uint64_t size); void zfs_range_tree_adjust_fill(zfs_range_tree_t *rt, zfs_range_seg_t *rs, diff --git a/include/sys/space_map.h b/include/sys/space_map.h index 6363f347c591..04c6795e9ed7 100644 --- a/include/sys/space_map.h +++ b/include/sys/space_map.h @@ -208,9 +208,22 @@ boolean_t sm_entry_is_double_word(uint64_t e); typedef int (*sm_cb_t)(space_map_entry_t *sme, void *arg); +typedef struct space_map_load_result { + /* Allocated bytes reconstructed from the entries that were loaded. */ + uint64_t smlr_allocated; + uint64_t smlr_repaired_entries; + uint64_t smlr_affected_bytes; + space_map_entry_t smlr_first_entry; + space_map_entry_t smlr_last_entry; + uint64_t smlr_first_free_bytes; + uint64_t smlr_last_free_bytes; +} space_map_load_result_t; + int space_map_load(space_map_t *sm, zfs_range_tree_t *rt, maptype_t maptype); int space_map_load_length(space_map_t *sm, zfs_range_tree_t *rt, maptype_t maptype, uint64_t length); +int space_map_load_length_repair(space_map_t *sm, zfs_range_tree_t *rt, + uint64_t length, space_map_load_result_t *result); int space_map_iterate(space_map_t *sm, uint64_t length, sm_cb_t callback, void *arg); int space_map_incremental_destroy(space_map_t *sm, sm_cb_t callback, void *arg, diff --git a/module/zfs/metaslab.c b/module/zfs/metaslab.c index f29bdc0118b8..e634939f9136 100644 --- a/module/zfs/metaslab.c +++ b/module/zfs/metaslab.c @@ -366,6 +366,10 @@ typedef struct metaslab_stats { kstat_named_t metaslabstat_reload_tree; kstat_named_t metaslabstat_too_many_tries; kstat_named_t metaslabstat_try_hard; + kstat_named_t metaslabstat_load_repairs; + kstat_named_t metaslabstat_load_repaired_entries; + kstat_named_t metaslabstat_load_repaired_bytes; + kstat_named_t metaslabstat_load_unloadable; } metaslab_stats_t; static metaslab_stats_t metaslab_stats = { @@ -373,10 +377,16 @@ static metaslab_stats_t metaslab_stats = { { "reload_tree", KSTAT_DATA_UINT64 }, { "too_many_tries", KSTAT_DATA_UINT64 }, { "try_hard", KSTAT_DATA_UINT64 }, + { "load_repairs", KSTAT_DATA_UINT64 }, + { "load_repaired_entries", KSTAT_DATA_UINT64 }, + { "load_repaired_bytes", KSTAT_DATA_UINT64 }, + { "load_unloadable", KSTAT_DATA_UINT64 }, }; #define METASLABSTAT_BUMP(stat) \ atomic_inc_64(&metaslab_stats.stat.value.ui64); +#define METASLABSTAT_ADD(stat, amount) \ + atomic_add_64(&metaslab_stats.stat.value.ui64, amount); char * metaslab_rt_name(metaslab_group_t *mg, metaslab_t *ms, const char *name) @@ -2159,20 +2169,31 @@ metaslab_verify_space(metaslab_t *msp, uint64_t txg) !msp->ms_loaded) return; - /* - * Even though the smp_alloc field can get negative, - * when it comes to a metaslab's space map, that should - * never be the case. - */ - ASSERT3S(space_map_allocated(msp->ms_sm), >=, 0); - - ASSERT3U(space_map_allocated(msp->ms_sm), >=, + uint64_t expected_allocated; + if (msp->ms_repair.msr_entries != 0 || + msp->ms_smp_alloc_invalid) { + expected_allocated = msp->ms_repair.msr_allocated; + } else { + /* + * Even though smp_alloc can be negative for other space + * maps, a healthy metaslab space map must be in range. + */ + ASSERT3S(space_map_allocated(msp->ms_sm), >=, 0); + ASSERT3U(space_map_allocated(msp->ms_sm), <=, msp->ms_size); + expected_allocated = + (uint64_t)space_map_allocated(msp->ms_sm); + } + ASSERT3U(expected_allocated, >=, zfs_range_tree_space(msp->ms_unflushed_frees)); - - ASSERT3U(metaslab_allocated_space(msp), ==, - space_map_allocated(msp->ms_sm) + - zfs_range_tree_space(msp->ms_unflushed_allocs) - + ASSERT3U(expected_allocated, <=, UINT64_MAX - + zfs_range_tree_space(msp->ms_unflushed_allocs)); + expected_allocated += + zfs_range_tree_space(msp->ms_unflushed_allocs); + ASSERT3U(expected_allocated, >=, zfs_range_tree_space(msp->ms_unflushed_frees)); + expected_allocated -= + zfs_range_tree_space(msp->ms_unflushed_frees); + ASSERT3U(metaslab_allocated_space(msp), ==, expected_allocated); sm_free_space = msp->ms_size - metaslab_allocated_space(msp); @@ -2199,6 +2220,119 @@ metaslab_verify_space(metaslab_t *msp, uint64_t txg) VERIFY3U(sm_free_space, ==, msp_free_space); } +static int +metaslab_reconcile_load_repair(metaslab_t *msp, + const space_map_load_result_t *result, boolean_t repair_needed, + int64_t *alloc_delta) +{ + uint64_t old_allocated = msp->ms_allocated_space; + uint64_t new_allocated = repair_needed ? + result->smlr_allocated : old_allocated; + + if (new_allocated > msp->ms_size) + return (SET_ERROR(ECKSUM)); + + uint64_t magnitude; + if (new_allocated >= old_allocated) { + magnitude = new_allocated - old_allocated; + if (magnitude > INT64_MAX) + return (SET_ERROR(ECKSUM)); + *alloc_delta = (int64_t)magnitude; + } else { + magnitude = old_allocated - new_allocated; + if (magnitude > INT64_MAX) + return (SET_ERROR(ECKSUM)); + *alloc_delta = -(int64_t)magnitude; + } + + msp->ms_allocated_space = new_allocated; + return (0); +} + +static void +metaslab_repair_clear(metaslab_t *msp) +{ + memset(&msp->ms_repair, 0, sizeof (msp->ms_repair)); +} + +static boolean_t +metaslab_load_has_pending_changes(metaslab_t *msp) +{ + if (msp->ms_allocating_total != 0 || + msp->ms_allocated_this_txg != 0 || + msp->ms_deferspace != 0 || + !zfs_range_tree_is_empty(msp->ms_unflushed_allocs) || + !zfs_range_tree_is_empty(msp->ms_unflushed_frees) || + !zfs_range_tree_is_empty(msp->ms_freeing) || + !zfs_range_tree_is_empty(msp->ms_freed) || + !zfs_range_tree_is_empty(msp->ms_checkpointing)) + return (B_TRUE); + + for (int t = 0; t < TXG_SIZE; t++) { + if (!zfs_range_tree_is_empty(msp->ms_allocating[t])) + return (B_TRUE); + } + for (int t = 0; t < TXG_DEFER_SIZE; t++) { + if (!zfs_range_tree_is_empty(msp->ms_defer[t])) + return (B_TRUE); + } + + return (B_FALSE); +} + +static void +metaslab_account_unloadable(metaslab_t *msp) +{ + ASSERT(MUTEX_HELD(&msp->ms_lock)); + ASSERT3U(msp->ms_allocated_space, <=, msp->ms_size); + + /* + * Once all known changes have been accounted, charge the remaining + * capacity as allocated so an unreadable metaslab is never reported as + * usable space. + */ + uint64_t unavailable = msp->ms_size - msp->ms_allocated_space; + while (unavailable != 0) { + int64_t delta = (int64_t)MIN(unavailable, + (uint64_t)INT64_MAX); + metaslab_space_update(msp->ms_group, delta, 0, 0); + unavailable -= (uint64_t)delta; + } + msp->ms_allocated_space = msp->ms_size; +} + +static void +metaslab_mark_unloadable(metaslab_t *msp) +{ + ASSERT(MUTEX_HELD(&msp->ms_lock)); + + boolean_t pending = metaslab_load_has_pending_changes(msp); + if (!pending) + metaslab_account_unloadable(msp); + + metaslab_repair_clear(msp); + msp->ms_load_state = METASLAB_LOAD_UNLOADABLE; + msp->ms_condense_wanted = B_FALSE; + msp->ms_max_size = 0; + zfs_range_tree_vacate(msp->ms_trim, NULL, NULL); + metaslab_group_sort(msp->ms_group, msp, 0); + METASLABSTAT_BUMP(metaslabstat_load_unloadable); + + spa_t *spa = msp->ms_group->mg_vd->vdev_spa; + uint64_t txg = spa_syncing_txg(spa); + if (pending && spa_writeable(spa) && + txg < spa_final_dirty_txg(spa)) { + vdev_dirty(msp->ms_group->mg_vd, VDD_METASLAB, msp, txg + 1); + } + + zfs_dbgmsg("metaslab_load: quarantining unloadable metaslab, " + "spa %s, vdev_id %llu, ms_id %llu, object %llu", + spa_name(msp->ms_group->mg_vd->vdev_spa), + (u_longlong_t)msp->ms_group->mg_vd->vdev_id, + (u_longlong_t)msp->ms_id, + (u_longlong_t)space_map_object(msp->ms_sm)); +} + static void metaslab_aux_histograms_clear(metaslab_t *msp) { @@ -2334,6 +2468,9 @@ metaslab_verify_weight_and_frag(metaslab_t *msp) if (msp->ms_group == NULL) return; + if (msp->ms_load_state != METASLAB_LOAD_NORMAL) + return; + /* * Devices being removed always return a weight of 0 and leave * fragmentation and ms_max_size as is - there is nothing for @@ -2495,6 +2632,7 @@ static int metaslab_load_impl(metaslab_t *msp) { int error = 0; + space_map_load_result_t load_result = { 0 }; ASSERT(MUTEX_HELD(&msp->ms_lock)); ASSERT(msp->ms_loading); @@ -2540,8 +2678,8 @@ metaslab_load_impl(metaslab_t *msp) mrap->mra_floor_shift = metaslab_by_size_min_shift; if (msp->ms_sm != NULL) { - error = space_map_load_length(msp->ms_sm, msp->ms_allocatable, - SM_FREE, length); + error = space_map_load_length_repair(msp->ms_sm, + msp->ms_allocatable, length, &load_result); /* Now, populate the size-sorted tree. */ metaslab_rt_create(msp->ms_allocatable, mrap); @@ -2603,7 +2741,108 @@ metaslab_load_impl(metaslab_t *msp) } ASSERT3P(msp->ms_group, !=, NULL); + metaslab_group_t *mg = msp->ms_group; + vdev_t *vd = mg->mg_vd; + spa_t *spa = vd->vdev_spa; + int64_t alloc_delta; + boolean_t repair_needed = load_result.smlr_repaired_entries != 0 || + msp->ms_smp_alloc_invalid; + + /* + * Repair is only self-contained when no changes outside the loaded + * space map must be merged into ms_allocatable. If there are pending + * changes, defer the load until syncing drains them rather than risk + * making an ambiguous range allocatable again. + */ + if (repair_needed && metaslab_load_has_pending_changes(msp)) { + zfs_dbgmsg("metaslab_load: cannot safely persist overlap " + "repair with pending changes, spa %s, vdev_id %llu, " + "ms_id %llu, object %llu", + spa_name(spa), (u_longlong_t)vd->vdev_id, + (u_longlong_t)msp->ms_id, + (u_longlong_t)space_map_object(msp->ms_sm)); + zfs_range_tree_vacate(msp->ms_allocatable, NULL, NULL); + msp->ms_load_state = METASLAB_LOAD_DEFERRED; + msp->ms_condense_wanted = B_FALSE; + metaslab_group_sort(mg, msp, 0); + uint64_t txg = spa_syncing_txg(spa); + if (spa_writeable(spa) && txg < spa_final_dirty_txg(spa)) + vdev_dirty(vd, VDD_METASLAB, msp, txg + 1); + mutex_exit(&msp->ms_sync_lock); + return (SET_ERROR(EAGAIN)); + } + + error = metaslab_reconcile_load_repair(msp, &load_result, + repair_needed, &alloc_delta); + if (error != 0) { + zfs_range_tree_vacate(msp->ms_allocatable, NULL, NULL); + mutex_exit(&msp->ms_sync_lock); + return (error); + } + msp->ms_loaded = B_TRUE; + msp->ms_load_state = METASLAB_LOAD_NORMAL; + if (alloc_delta != 0) + metaslab_space_update(mg, alloc_delta, 0, 0); + + if (repair_needed) { + msp->ms_repair = (metaslab_repair_t) { + .msr_smp_alloc = space_map_allocated(msp->ms_sm), + .msr_allocated = load_result.smlr_allocated, + .msr_entries = load_result.smlr_repaired_entries, + .msr_bytes = load_result.smlr_affected_bytes, + }; + + uint64_t txg = spa_syncing_txg(spa); + if (spa_writeable(spa) && txg < spa_final_dirty_txg(spa)) { + msp->ms_condense_wanted = B_TRUE; + vdev_dirty(vd, VDD_METASLAB, msp, txg + 1); + } + + METASLABSTAT_BUMP(metaslabstat_load_repairs); + METASLABSTAT_ADD(metaslabstat_load_repaired_entries, + load_result.smlr_repaired_entries); + METASLABSTAT_ADD(metaslabstat_load_repaired_bytes, + load_result.smlr_affected_bytes); + zfs_dbgmsg("metaslab_load: repaired invalid_smp_alloc=%u and " + "quarantined %llu overlapping space map entries " + "affecting %llu bytes, smp_alloc %lld, reconstructed " + "allocation %llu, spa %s, vdev_id %llu, ms_id %llu, " + "object %llu", + msp->ms_smp_alloc_invalid, + (u_longlong_t)load_result.smlr_repaired_entries, + (u_longlong_t)load_result.smlr_affected_bytes, + (longlong_t)msp->ms_repair.msr_smp_alloc, + (u_longlong_t)load_result.smlr_allocated, spa_name(spa), + (u_longlong_t)vd->vdev_id, (u_longlong_t)msp->ms_id, + (u_longlong_t)space_map_object(msp->ms_sm)); + if (load_result.smlr_repaired_entries != 0) { + zfs_dbgmsg("metaslab_load: first %s offset=%llx " + "run=%llx free_before=%llx txg=%llu pass=%llu; " + "last %s offset=%llx run=%llx free_before=%llx " + "txg=%llu pass=%llu", + load_result.smlr_first_entry.sme_type == SM_ALLOC ? + "ALLOC" : "FREE", + (u_longlong_t) + load_result.smlr_first_entry.sme_offset, + (u_longlong_t) + load_result.smlr_first_entry.sme_run, + (u_longlong_t)load_result.smlr_first_free_bytes, + (u_longlong_t) + load_result.smlr_first_entry.sme_txg, + (u_longlong_t) + load_result.smlr_first_entry.sme_sync_pass, + load_result.smlr_last_entry.sme_type == SM_ALLOC ? + "ALLOC" : "FREE", + (u_longlong_t) + load_result.smlr_last_entry.sme_offset, + (u_longlong_t)load_result.smlr_last_entry.sme_run, + (u_longlong_t)load_result.smlr_last_free_bytes, + (u_longlong_t)load_result.smlr_last_entry.sme_txg, + (u_longlong_t) + load_result.smlr_last_entry.sme_sync_pass); + } + } /* * Apply all the unflushed changes to ms_allocatable right @@ -2616,7 +2855,6 @@ metaslab_load_impl(metaslab_t *msp) zfs_range_tree_add, msp->ms_allocatable); ASSERT3P(msp->ms_group, !=, NULL); - spa_t *spa = msp->ms_group->mg_vd->vdev_spa; if (spa_syncing_log_sm(spa) != NULL) { ASSERT(spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP)); @@ -2679,10 +2917,12 @@ metaslab_load_impl(metaslab_t *msp) uint64_t weight = msp->ms_weight; uint64_t max_size = msp->ms_max_size; metaslab_recalculate_weight_and_sort(msp); - if (!WEIGHT_IS_SPACEBASED(weight)) + if (!WEIGHT_IS_SPACEBASED(weight) && + load_result.smlr_repaired_entries == 0) ASSERT3U(weight, <=, msp->ms_weight); msp->ms_max_size = metaslab_largest_allocatable(msp); - ASSERT3U(max_size, <=, msp->ms_max_size); + if (load_result.smlr_repaired_entries == 0) + ASSERT3U(max_size, <=, msp->ms_max_size); hrtime_t load_end = gethrtime(); msp->ms_load_time = load_end; zfs_dbgmsg("metaslab_load: txg %llu, spa %s, class %s, vdev_id %llu, " @@ -2725,6 +2965,10 @@ metaslab_load(metaslab_t *msp) metaslab_load_wait(msp); if (msp->ms_loaded) return (0); + if (msp->ms_load_state == METASLAB_LOAD_DEFERRED) + return (SET_ERROR(EAGAIN)); + if (msp->ms_load_state == METASLAB_LOAD_UNLOADABLE) + return (SET_ERROR(ECKSUM)); VERIFY(!msp->ms_loading); ASSERT(!msp->ms_condensing); @@ -2764,6 +3008,8 @@ metaslab_load(metaslab_t *msp) int error = metaslab_load_impl(msp); ASSERT(MUTEX_HELD(&msp->ms_lock)); + if (error == ECKSUM) + metaslab_mark_unloadable(msp); msp->ms_loading = B_FALSE; cv_broadcast(&msp->ms_load_cv); @@ -2942,7 +3188,20 @@ metaslab_init(metaslab_group_t *mg, uint64_t id, uint64_t object, } ASSERT(ms->ms_sm != NULL); - ms->ms_allocated_space = space_map_allocated(ms->ms_sm); + int64_t allocated = space_map_allocated(ms->ms_sm); + if (allocated < 0 || (uint64_t)allocated > ms->ms_size) { + ms->ms_allocated_space = ms->ms_size; + ms->ms_smp_alloc_invalid = B_TRUE; + zfs_dbgmsg("metaslab_init: invalid allocated space " + "%lld in spa %s, vdev_id %llu, ms_id %llu, " + "object %llu", + (longlong_t)allocated, spa_name(spa), + (u_longlong_t)vd->vdev_id, + (u_longlong_t)ms->ms_id, + (u_longlong_t)space_map_object(ms->ms_sm)); + } else { + ms->ms_allocated_space = (uint64_t)allocated; + } } uint64_t shift, start; @@ -3004,6 +3263,18 @@ metaslab_init(metaslab_group_t *mg, uint64_t id, uint64_t object, metaslab_space_update(mg, metaslab_allocated_space(ms), 0, 0); } + /* + * The conservative accounting above gives this metaslab zero weight, + * so explicitly schedule a preload to reconstruct its allocation count. + */ + if (ms->ms_smp_alloc_invalid && spa_writeable(spa)) { + uint64_t repair_txg = spa_syncing_txg(spa); + if (repair_txg < spa_final_dirty_txg(spa)) { + ms->ms_condense_wanted = B_TRUE; + vdev_dirty(vd, VDD_METASLAB, ms, repair_txg + 1); + } + } + if (txg != 0) { vdev_dirty(vd, 0, NULL, txg); vdev_dirty(vd, VDD_METASLAB, ms, txg); @@ -3469,6 +3740,9 @@ metaslab_segment_weight(metaslab_t *msp) static boolean_t metaslab_should_allocate(metaslab_t *msp, uint64_t asize, boolean_t try_hard) { + if (unlikely(msp->ms_load_state != METASLAB_LOAD_NORMAL)) + return (B_FALSE); + /* * This case will usually but not always get caught by the checks below; * metaslabs can be loaded by various means, including the trim and @@ -3517,6 +3791,9 @@ metaslab_weight(metaslab_t *msp, boolean_t nodirty) ASSERT(MUTEX_HELD(&msp->ms_lock)); + if (msp->ms_load_state != METASLAB_LOAD_NORMAL) + return (0); + metaslab_set_fragmentation(msp, nodirty); /* @@ -3785,8 +4062,8 @@ metaslab_preload(void *arg) ASSERT(!MUTEX_HELD(&msp->ms_group->mg_lock)); mutex_enter(&msp->ms_lock); - (void) metaslab_load(msp); - metaslab_set_selected_txg(msp, spa_syncing_txg(spa)); + if (metaslab_load(msp) == 0) + metaslab_set_selected_txg(msp, spa_syncing_txg(spa)); mutex_exit(&msp->ms_lock); spl_fstrans_unmark(cookie); } @@ -3893,6 +4170,8 @@ metaslab_condense(metaslab_t *msp, dmu_tx_t *tx) space_map_t *sm = msp->ms_sm; uint64_t txg = dmu_tx_get_txg(tx); spa_t *spa = msp->ms_group->mg_vd->vdev_spa; + metaslab_repair_t repair = msp->ms_repair; + boolean_t repair_summary = msp->ms_smp_alloc_invalid; ASSERT(MUTEX_HELD(&msp->ms_lock)); ASSERT(msp->ms_loaded); @@ -4032,10 +4311,24 @@ metaslab_condense(metaslab_t *msp, dmu_tx_t *tx) zfs_range_tree_destroy(condense_tree); zfs_range_tree_vacate(tmp_tree, NULL, NULL); zfs_range_tree_destroy(tmp_tree); + if (repair.msr_entries != 0 || repair_summary) { + spa_history_log_internal(spa, "metaslab repair", tx, + "vdev=%llu metaslab=%llu object=%llu entries=%llu " + "bytes=%llu smp_alloc=%lld reconstructed_alloc=%llu " + "invalid_smp_alloc=%u", + (u_longlong_t)msp->ms_group->mg_vd->vdev_id, + (u_longlong_t)msp->ms_id, (u_longlong_t)object, + (u_longlong_t)repair.msr_entries, + (u_longlong_t)repair.msr_bytes, + (longlong_t)repair.msr_smp_alloc, + (u_longlong_t)repair.msr_allocated, repair_summary); + } mutex_enter(&msp->ms_lock); msp->ms_condensing = B_FALSE; metaslab_flush_update(msp, tx); + metaslab_repair_clear(msp); + msp->ms_smp_alloc_invalid = B_FALSE; } static void @@ -4249,6 +4542,10 @@ metaslab_flush(metaslab_t *msp, dmu_tx_t *tx) zfs_range_tree_vacate(msp->ms_unflushed_allocs, NULL, NULL); zfs_range_tree_vacate(msp->ms_unflushed_frees, NULL, NULL); + if (msp->ms_load_state == METASLAB_LOAD_UNLOADABLE && + !metaslab_load_has_pending_changes(msp)) + metaslab_account_unloadable(msp); + metaslab_verify_space(msp, dmu_tx_get_txg(tx)); metaslab_verify_weight_and_frag(msp); @@ -4303,6 +4600,8 @@ metaslab_sync(metaslab_t *msp, uint64_t txg) if (zfs_range_tree_is_empty(alloctree) && zfs_range_tree_is_empty(msp->ms_freeing) && zfs_range_tree_is_empty(msp->ms_checkpointing) && + !(msp->ms_load_state != METASLAB_LOAD_NORMAL && + txg <= spa_final_dirty_txg(spa)) && !(msp->ms_loaded && msp->ms_condense_wanted && txg <= spa_final_dirty_txg(spa))) return; @@ -4375,6 +4674,29 @@ metaslab_sync(metaslab_t *msp, uint64_t txg) mutex_enter(&msp->ms_sync_lock); mutex_enter(&msp->ms_lock); + /* + * A repair load can be deferred by changes which still live in the + * pool-wide log space map. Force those changes into this metaslab's + * space map so the next preload can repair and condense it. + */ + if (spa_sync_pass(spa) == 1 && + msp->ms_load_state != METASLAB_LOAD_NORMAL && + spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP) && + metaslab_unflushed_txg(msp) != 0 && + metaslab_unflushed_txg(msp) < txg && + metaslab_unflushed_dirty(msp)) { + boolean_t flush_only = zfs_range_tree_is_empty(alloctree) && + zfs_range_tree_is_empty(msp->ms_freeing) && + zfs_range_tree_is_empty(msp->ms_checkpointing); + + if (metaslab_flush(msp, tx) && flush_only) { + mutex_exit(&msp->ms_lock); + mutex_exit(&msp->ms_sync_lock); + dmu_tx_commit(tx); + return; + } + } + /* * Note: metaslab_condense() clears the space map's histogram. * Therefore we must verify and remove this histogram before @@ -4575,6 +4897,7 @@ metaslab_sync_done(metaslab_t *msp, uint64_t txg) zfs_range_tree_t **defer_tree; int64_t alloc_delta, defer_delta; boolean_t defer_allowed = B_TRUE; + boolean_t retry_load = B_FALSE; ASSERT(!vd->vdev_ishole); @@ -4634,7 +4957,8 @@ metaslab_sync_done(metaslab_t *msp, uint64_t txg) * can be discarded at any time with the sole consequence of recent * frees not being trimmed. */ - if (spa_get_autotrim(spa) == SPA_AUTOTRIM_ON) { + if (spa_get_autotrim(spa) == SPA_AUTOTRIM_ON && + msp->ms_load_state != METASLAB_LOAD_UNLOADABLE) { zfs_range_tree_walk(*defer_tree, zfs_range_tree_add, msp->ms_trim); if (!defer_allowed) { @@ -4694,7 +5018,28 @@ metaslab_sync_done(metaslab_t *msp, uint64_t txg) ASSERT0(zfs_range_tree_space(msp->ms_checkpointing)); msp->ms_allocating_total -= msp->ms_allocated_this_txg; msp->ms_allocated_this_txg = 0; + if (msp->ms_load_state == METASLAB_LOAD_DEFERRED) { + if (!metaslab_load_has_pending_changes(msp)) { + msp->ms_load_state = METASLAB_LOAD_NORMAL; + metaslab_recalculate_weight_and_sort(msp); + retry_load = B_TRUE; + } else if (txg < spa_final_dirty_txg(spa)) { + vdev_dirty(vd, VDD_METASLAB, msp, txg + 1); + } + } + if (msp->ms_load_state == METASLAB_LOAD_UNLOADABLE) { + if (!metaslab_load_has_pending_changes(msp)) { + metaslab_account_unloadable(msp); + } else if (txg < spa_final_dirty_txg(spa)) { + vdev_dirty(vd, VDD_METASLAB, msp, txg + 1); + } + } mutex_exit(&msp->ms_lock); + + if (retry_load && !spa_shutting_down(spa)) { + VERIFY(taskq_dispatch(spa->spa_metaslab_taskq, + metaslab_preload, msp, TQ_SLEEP) != TASKQID_INVALID); + } } void diff --git a/module/zfs/range_tree.c b/module/zfs/range_tree.c index 916889c3130c..b937225f9783 100644 --- a/module/zfs/range_tree.c +++ b/module/zfs/range_tree.c @@ -325,8 +325,9 @@ zfs_range_tree_adjust_fill(zfs_range_tree_t *rt, zfs_range_seg_t *rs, rt->rt_ops->rtop_add(rt, rs, rt->rt_arg); } -static void -zfs_range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill) +static boolean_t +zfs_range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill, + boolean_t panic_on_error) { zfs_range_tree_t *rt = arg; zfs_btree_index_t where; @@ -356,17 +357,20 @@ zfs_range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill) uint64_t rstart = zfs_rs_get_start(rs, rt); uint64_t rend = zfs_rs_get_end(rs, rt); if (gap == 0) { - zfs_panic_recover("zfs: rt=%s: adding segment " - "(offset=%llx size=%llx) overlapping with existing " - "one (offset=%llx size=%llx)", - ZFS_RT_NAME(rt), - (longlong_t)start, (longlong_t)size, - (longlong_t)rstart, (longlong_t)(rend - rstart)); - return; + if (panic_on_error) { + zfs_panic_recover("zfs: rt=%s: adding segment " + "(offset=%llx size=%llx) overlapping with " + "existing one (offset=%llx size=%llx)", + ZFS_RT_NAME(rt), + (longlong_t)start, (longlong_t)size, + (longlong_t)rstart, + (longlong_t)(rend - rstart)); + } + return (B_FALSE); } if (rstart <= start && rend >= end) { zfs_range_tree_adjust_fill(rt, rs, fill); - return; + return (B_TRUE); } if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL) @@ -381,8 +385,8 @@ zfs_range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill) size = end - start; zfs_btree_remove(&rt->rt_root, rs); - zfs_range_tree_add_impl(rt, start, size, fill); - return; + return (zfs_range_tree_add_impl(rt, start, size, fill, + panic_on_error)); } ASSERT0P(rs); @@ -472,17 +476,29 @@ zfs_range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill) zfs_range_tree_stat_incr(rt, rs); rt->rt_space += size + bridge_size; + return (B_TRUE); } void zfs_range_tree_add(void *arg, uint64_t start, uint64_t size) { - zfs_range_tree_add_impl(arg, start, size, size); + (void) zfs_range_tree_add_impl(arg, start, size, size, B_TRUE); } -static void +boolean_t +zfs_range_tree_try_add(zfs_range_tree_t *rt, uint64_t start, uint64_t size) +{ + /* + * Gap trees support overlapping additions by adjusting their fill, + * so only gap-free trees have a meaningful non-overlapping add. + */ + ASSERT0(rt->rt_gap); + return (zfs_range_tree_add_impl(rt, start, size, size, B_FALSE)); +} + +static boolean_t zfs_range_tree_remove_impl(zfs_range_tree_t *rt, uint64_t start, uint64_t size, - boolean_t do_fill) + boolean_t do_fill, boolean_t panic_on_error) { zfs_btree_index_t where; zfs_range_seg_t *rs; @@ -492,7 +508,11 @@ zfs_range_tree_remove_impl(zfs_range_tree_t *rt, uint64_t start, uint64_t size, boolean_t left_over, right_over; VERIFY3U(size, !=, 0); - VERIFY3U(size, <=, rt->rt_space); + if (size > rt->rt_space) { + if (panic_on_error) + VERIFY3U(size, <=, rt->rt_space); + return (B_FALSE); + } if (rt->rt_type == ZFS_RANGE_SEG64) ASSERT3U(start + size, >, start); @@ -502,10 +522,13 @@ zfs_range_tree_remove_impl(zfs_range_tree_t *rt, uint64_t start, uint64_t size, /* Make sure we completely overlap with someone */ if (rs == NULL) { - zfs_panic_recover("zfs: rt=%s: removing nonexistent segment " - "from range tree (offset=%llx size=%llx)", - ZFS_RT_NAME(rt), (longlong_t)start, (longlong_t)size); - return; + if (panic_on_error) { + zfs_panic_recover("zfs: rt=%s: removing nonexistent " + "segment from range tree (offset=%llx size=%llx)", + ZFS_RT_NAME(rt), (longlong_t)start, + (longlong_t)size); + } + return (B_FALSE); } rstart = zfs_rs_get_start(rs, rt); @@ -525,27 +548,33 @@ zfs_range_tree_remove_impl(zfs_range_tree_t *rt, uint64_t start, uint64_t size, size = end - start; } else { zfs_range_tree_adjust_fill(rt, rs, -size); - return; + return (B_TRUE); } } else if (rstart != start || rend != end) { - zfs_panic_recover("zfs: rt=%s: freeing partial segment " - "of gap tree (offset=%llx size=%llx) of " - "(offset=%llx size=%llx)", - ZFS_RT_NAME(rt), - (longlong_t)start, (longlong_t)size, - (longlong_t)rstart, (longlong_t)(rend - rstart)); - return; + if (panic_on_error) { + zfs_panic_recover("zfs: rt=%s: freeing partial " + "segment of gap tree (offset=%llx " + "size=%llx) of (offset=%llx size=%llx)", + ZFS_RT_NAME(rt), + (longlong_t)start, (longlong_t)size, + (longlong_t)rstart, + (longlong_t)(rend - rstart)); + } + return (B_FALSE); } } if (!(rstart <= start && rend >= end)) { - zfs_panic_recover("zfs: rt=%s: removing segment " - "(offset=%llx size=%llx) not completely overlapped by " - "existing one (offset=%llx size=%llx)", - ZFS_RT_NAME(rt), - (longlong_t)start, (longlong_t)size, - (longlong_t)rstart, (longlong_t)(rend - rstart)); - return; + if (panic_on_error) { + zfs_panic_recover("zfs: rt=%s: removing segment " + "(offset=%llx size=%llx) not completely " + "overlapped by existing one (offset=%llx " + "size=%llx)", ZFS_RT_NAME(rt), + (longlong_t)start, (longlong_t)size, + (longlong_t)rstart, + (longlong_t)(rend - rstart)); + } + return (B_FALSE); } left_over = (rstart != start); @@ -602,18 +631,30 @@ zfs_range_tree_remove_impl(zfs_range_tree_t *rt, uint64_t start, uint64_t size, } rt->rt_space -= size; + return (B_TRUE); } void zfs_range_tree_remove(void *arg, uint64_t start, uint64_t size) { - zfs_range_tree_remove_impl(arg, start, size, B_FALSE); + (void) zfs_range_tree_remove_impl(arg, start, size, B_FALSE, B_TRUE); +} + +boolean_t +zfs_range_tree_try_remove(zfs_range_tree_t *rt, uint64_t start, uint64_t size) +{ + /* + * Return without modifying the tree unless one segment completely + * contains the requested range. + */ + ASSERT0(rt->rt_gap); + return (zfs_range_tree_remove_impl(rt, start, size, B_FALSE, B_FALSE)); } void zfs_range_tree_remove_fill(zfs_range_tree_t *rt, uint64_t start, uint64_t size) { - zfs_range_tree_remove_impl(rt, start, size, B_TRUE); + (void) zfs_range_tree_remove_impl(rt, start, size, B_TRUE, B_TRUE); } void diff --git a/module/zfs/spa_log_spacemap.c b/module/zfs/spa_log_spacemap.c index 656f15b243a4..cf754b27512d 100644 --- a/module/zfs/spa_log_spacemap.c +++ b/module/zfs/spa_log_spacemap.c @@ -1437,14 +1437,19 @@ spa_ld_log_sm_data(spa_t *spa) for (metaslab_t *m = avl_first(&spa->spa_metaslabs_by_flushed); m != NULL; m = AVL_NEXT(&spa->spa_metaslabs_by_flushed, m)) { mutex_enter(&m->ms_lock); - m->ms_allocated_space = space_map_allocated(m->ms_sm) + - zfs_range_tree_space(m->ms_unflushed_allocs) - - zfs_range_tree_space(m->ms_unflushed_frees); - - metaslab_space_update(m->ms_group, - zfs_range_tree_space(m->ms_unflushed_allocs), 0, 0); - metaslab_space_update(m->ms_group, - -zfs_range_tree_space(m->ms_unflushed_frees), 0, 0); + if (m->ms_load_state != METASLAB_LOAD_UNLOADABLE && + !m->ms_smp_alloc_invalid) { + m->ms_allocated_space = space_map_allocated(m->ms_sm) + + zfs_range_tree_space(m->ms_unflushed_allocs) - + zfs_range_tree_space(m->ms_unflushed_frees); + + metaslab_space_update(m->ms_group, + zfs_range_tree_space(m->ms_unflushed_allocs), + 0, 0); + metaslab_space_update(m->ms_group, + -zfs_range_tree_space(m->ms_unflushed_frees), + 0, 0); + } ASSERT0(m->ms_weight & METASLAB_ACTIVE_MASK); metaslab_recalculate_weight_and_sort(m); @@ -1453,8 +1458,8 @@ spa_ld_log_sm_data(spa_t *spa) metaslab_unflushed_changes_memused(m); if (metaslab_debug_load && m->ms_sm != NULL) { - VERIFY0(metaslab_load(m)); - metaslab_set_selected_txg(m, 0); + if (metaslab_load(m) == 0) + metaslab_set_selected_txg(m, 0); } mutex_exit(&m->ms_lock); } diff --git a/module/zfs/space_map.c b/module/zfs/space_map.c index 13c606e9ff34..746c730aa9ac 100644 --- a/module/zfs/space_map.c +++ b/module/zfs/space_map.c @@ -82,9 +82,11 @@ sm_entry_is_double_word(uint64_t e) /* * Iterate through the space map, invoking the callback on each (non-debug) * space map entry. Stop after reading 'end' bytes of the space map. + * When repairing, return ECKSUM instead of asserting on malformed entries. */ -int -space_map_iterate(space_map_t *sm, uint64_t end, sm_cb_t callback, void *arg) +static int +space_map_iterate_impl(space_map_t *sm, uint64_t end, sm_cb_t callback, + void *arg, boolean_t repair) { uint64_t blksz = sm->sm_blksz; @@ -156,13 +158,24 @@ space_map_iterate(space_map_t *sm, uint64_t end, sm_cb_t callback, void *arg) /* move on to the second word */ block_cursor++; + if (repair && block_cursor >= block_end) { + error = SET_ERROR(ECKSUM); + break; + } + VERIFY3P(block_cursor, <, block_end); e = *block_cursor; - VERIFY3P(block_cursor, <=, block_end); type = SM2_TYPE_DECODE(e); raw_offset = SM2_OFFSET_DECODE(e); } + uint64_t map_units = sm->sm_size >> sm->sm_shift; + if (repair && (raw_offset >= map_units || + raw_run > map_units - raw_offset)) { + error = SET_ERROR(ECKSUM); + break; + } + uint64_t entry_offset = (raw_offset << sm->sm_shift) + sm->sm_start; uint64_t entry_run = raw_run << sm->sm_shift; @@ -190,6 +203,12 @@ space_map_iterate(space_map_t *sm, uint64_t end, sm_cb_t callback, void *arg) return (error); } +int +space_map_iterate(space_map_t *sm, uint64_t end, sm_cb_t callback, void *arg) +{ + return (space_map_iterate_impl(sm, end, callback, arg, B_FALSE)); +} + /* * Reads the entries from the last block of the space map into * buf in reverse order. Populates nwords with number of words @@ -394,13 +413,34 @@ space_map_incremental_destroy(space_map_t *sm, sm_cb_t callback, void *arg, typedef struct space_map_load_arg { space_map_t *smla_sm; zfs_range_tree_t *smla_rt; + zfs_range_tree_t *smla_repair_rt; maptype_t smla_type; + space_map_load_result_t *smla_result; } space_map_load_arg_t; +static void +space_map_load_record_repair(space_map_load_arg_t *smla, + const space_map_entry_t *sme, uint64_t free_bytes) +{ + space_map_load_result_t *result = smla->smla_result; + + ASSERT3P(result, !=, NULL); + ASSERT3U(free_bytes, <=, sme->sme_run); + + if (result->smlr_repaired_entries == 0) { + result->smlr_first_entry = *sme; + result->smlr_first_free_bytes = free_bytes; + } + result->smlr_last_entry = *sme; + result->smlr_last_free_bytes = free_bytes; + result->smlr_repaired_entries++; +} + static int space_map_load_callback(space_map_entry_t *sme, void *arg) { space_map_load_arg_t *smla = arg; + if (sme->sme_type == smla->smla_type) { VERIFY3U(zfs_range_tree_space(smla->smla_rt) + sme->sme_run, <=, smla->smla_sm->sm_size); @@ -414,33 +454,126 @@ space_map_load_callback(space_map_entry_t *sme, void *arg) return (0); } -/* - * Load the spacemap into the rangetree, like space_map_load. But only - * read the first 'length' bytes of the spacemap. - */ -int -space_map_load_length(space_map_t *sm, zfs_range_tree_t *rt, maptype_t maptype, - uint64_t length) +static int +space_map_load_repair_callback(space_map_entry_t *sme, void *arg) { - space_map_load_arg_t smla; + space_map_load_arg_t *smla = arg; + zfs_range_tree_t *rt = smla->smla_rt; + zfs_range_tree_t *repair_rt = smla->smla_repair_rt; + uint64_t overlap_start, overlap_size; + + ASSERT3P(smla->smla_result, !=, NULL); + ASSERT3S(smla->smla_type, ==, SM_FREE); + + boolean_t repair_overlap = repair_rt != NULL && + zfs_range_tree_find_in(repair_rt, sme->sme_offset, sme->sme_run, + &overlap_start, &overlap_size); + + if (!repair_overlap) { + if (sme->sme_type == SM_FREE) { + if (zfs_range_tree_try_add(rt, sme->sme_offset, + sme->sme_run)) + return (0); + } else { + if (zfs_range_tree_try_remove(rt, sme->sme_offset, + sme->sme_run)) + return (0); + } + } + + uint64_t free_before = zfs_range_tree_space(rt); + zfs_range_tree_clear(rt, sme->sme_offset, sme->sme_run); + free_before -= zfs_range_tree_space(rt); + /* + * An overlapping operation does not prove that any part of its range + * is unreferenced. Keep the complete range allocated for the rest of + * the replay and extend that quarantine through later overlapping + * entries. Condensation will persist this conservative state. + */ + if (repair_rt == NULL) { + repair_rt = zfs_range_tree_create_flags(NULL, rt->rt_type, NULL, + rt->rt_start, rt->rt_shift, 0, "space_map_load_repair"); + smla->smla_repair_rt = repair_rt; + } + zfs_range_tree_clear(repair_rt, sme->sme_offset, sme->sme_run); + zfs_range_tree_add(repair_rt, sme->sme_offset, sme->sme_run); + + space_map_load_record_repair(smla, sme, free_before); + return (0); +} + +static int +space_map_load_length_impl(space_map_t *sm, zfs_range_tree_t *rt, + maptype_t maptype, + uint64_t length, space_map_load_result_t *result) +{ + space_map_load_arg_t smla = { + .smla_sm = sm, + .smla_rt = rt, + .smla_type = maptype, + .smla_result = result, + }; VERIFY0(zfs_range_tree_space(rt)); if (maptype == SM_FREE) zfs_range_tree_add(rt, sm->sm_start, sm->sm_size); - smla.smla_rt = rt; - smla.smla_sm = sm; - smla.smla_type = maptype; - int err = space_map_iterate(sm, length, - space_map_load_callback, &smla); + if (result != NULL) { + memset(result, 0, sizeof (*result)); + } + + sm_cb_t callback = result == NULL ? space_map_load_callback : + space_map_load_repair_callback; + int err = space_map_iterate_impl(sm, length, callback, &smla, + result != NULL); + + if (result != NULL && err == 0) { + result->smlr_allocated = + sm->sm_size - zfs_range_tree_space(rt); + if (smla.smla_repair_rt != NULL) { + result->smlr_affected_bytes = + zfs_range_tree_space(smla.smla_repair_rt); + } + } if (err != 0) zfs_range_tree_vacate(rt, NULL, NULL); + if (smla.smla_repair_rt != NULL) { + zfs_range_tree_vacate(smla.smla_repair_rt, NULL, NULL); + zfs_range_tree_destroy(smla.smla_repair_rt); + } + return (err); } +/* + * Load the spacemap into the rangetree, like space_map_load. But only + * read the first 'length' bytes of the spacemap. + */ +int +space_map_load_length(space_map_t *sm, zfs_range_tree_t *rt, maptype_t maptype, + uint64_t length) +{ + return (space_map_load_length_impl(sm, rt, maptype, length, NULL)); +} + +/* + * Load a free-space map while conservatively quarantining entries that + * overlap the state reconstructed so far. The caller must persist the + * resulting allocation state before making the range tree allocatable. + */ +int +space_map_load_length_repair(space_map_t *sm, zfs_range_tree_t *rt, + uint64_t length, space_map_load_result_t *result) +{ + if (result == NULL) + return (SET_ERROR(EINVAL)); + + return (space_map_load_length_impl(sm, rt, SM_FREE, length, result)); +} + /* * Load the space map disk into the specified range tree. Segments of maptype * are added to the range tree, other segment types are removed. diff --git a/module/zfs/vdev_initialize.c b/module/zfs/vdev_initialize.c index b7709317df5e..5c431cd0ca64 100644 --- a/module/zfs/vdev_initialize.c +++ b/module/zfs/vdev_initialize.c @@ -448,7 +448,10 @@ vdev_initialize_calculate_progress(vdev_t *vd) * metaslab. Load it and walk the free tree for more accurate * progress estimation. */ - VERIFY0(metaslab_load(msp)); + if (metaslab_load(msp) != 0) { + mutex_exit(&msp->ms_lock); + continue; + } zfs_btree_index_t where; zfs_range_tree_t *rt = msp->ms_allocatable; @@ -574,7 +577,16 @@ vdev_initialize_thread(void *arg) mutex_enter(&msp->ms_lock); if (!msp->ms_loaded && !msp->ms_loading) unload_when_done = B_TRUE; - VERIFY0(metaslab_load(msp)); + error = metaslab_load(msp); + if (error != 0) { + mutex_exit(&msp->ms_lock); + metaslab_enable(msp, B_FALSE, unload_when_done); + spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); + zfs_dbgmsg("initialize: unable to load metaslab %llu " + "on vdev %s: error %d", + (u_longlong_t)msp->ms_id, vd->vdev_path, error); + break; + } zfs_range_tree_walk(msp->ms_allocatable, vdev_initialize_range_add, vd); @@ -603,7 +615,10 @@ vdev_initialize_thread(void *arg) mutex_enter(&vd->vdev_initialize_lock); if (!vd->vdev_initialize_exit_wanted) { - if (vdev_writeable(vd)) { + if (error != 0 && vdev_writeable(vd)) { + vdev_initialize_change_state(vd, + VDEV_INITIALIZE_SUSPENDED); + } else if (vdev_writeable(vd)) { vdev_initialize_change_state(vd, VDEV_INITIALIZE_COMPLETE); } else if (vd->vdev_faulted) { diff --git a/module/zfs/vdev_raidz.c b/module/zfs/vdev_raidz.c index 5fb568536691..96e235a93295 100644 --- a/module/zfs/vdev_raidz.c +++ b/module/zfs/vdev_raidz.c @@ -5009,7 +5009,7 @@ spa_raidz_expand_thread(void *arg, zthr_t *zthr) continue; } - VERIFY0(metaslab_load(msp)); + boolean_t loaded = (metaslab_load(msp) == 0); /* * We want to copy everything except the free (allocatable) @@ -5024,8 +5024,10 @@ spa_raidz_expand_thread(void *arg, zthr_t *zthr) metaslab_rt_name(msp->ms_group, msp, "spa_raidz_expand_thread:rt")); zfs_range_tree_add(rt, msp->ms_start, msp->ms_size); - zfs_range_tree_walk(msp->ms_allocatable, zfs_range_tree_remove, - rt); + if (loaded) { + zfs_range_tree_walk(msp->ms_allocatable, + zfs_range_tree_remove, rt); + } mutex_exit(&msp->ms_lock); /* diff --git a/module/zfs/vdev_trim.c b/module/zfs/vdev_trim.c index a97f6650a81c..3cfb0fefd764 100644 --- a/module/zfs/vdev_trim.c +++ b/module/zfs/vdev_trim.c @@ -728,7 +728,10 @@ vdev_trim_calculate_progress(vdev_t *vd) * metaslab. Load it and walk the free tree for more * accurate progress estimation. */ - VERIFY0(metaslab_load(msp)); + if (metaslab_load(msp) != 0) { + mutex_exit(&msp->ms_lock); + continue; + } zfs_range_tree_t *rt = msp->ms_allocatable; zfs_btree_t *bt = &rt->rt_root; @@ -857,10 +860,11 @@ vdev_trim_range_add(void *arg, uint64_t start, uint64_t size) */ if (zfs_flags & ZFS_DEBUG_TRIM) { metaslab_t *msp = ta->trim_msp; - VERIFY0(metaslab_load(msp)); + if (metaslab_load(msp) != 0) + return; VERIFY3B(msp->ms_loaded, ==, B_TRUE); - VERIFY(zfs_range_tree_contains(msp->ms_allocatable, start, - size)); + VERIFY(zfs_range_tree_contains(msp->ms_allocatable, + start, size)); } ASSERT(vd->vdev_ops->vdev_op_leaf); @@ -935,7 +939,16 @@ vdev_trim_thread(void *arg) spa_config_exit(spa, SCL_CONFIG, FTAG); metaslab_disable(msp); mutex_enter(&msp->ms_lock); - VERIFY0(metaslab_load(msp)); + error = metaslab_load(msp); + if (error != 0) { + mutex_exit(&msp->ms_lock); + metaslab_enable(msp, B_FALSE, B_FALSE); + spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); + zfs_dbgmsg("trim: unable to load metaslab %llu on " + "vdev %s: error %d", + (u_longlong_t)msp->ms_id, vd->vdev_path, error); + break; + } /* * If a partial TRIM was requested skip metaslabs which have @@ -970,7 +983,11 @@ vdev_trim_thread(void *arg) mutex_enter(&vd->vdev_trim_lock); if (!vd->vdev_trim_exit_wanted) { - if (vdev_writeable(vd)) { + if (error != 0 && vdev_writeable(vd)) { + vdev_trim_change_state(vd, VDEV_TRIM_SUSPENDED, + vd->vdev_trim_rate, vd->vdev_trim_partial, + vd->vdev_trim_secure); + } else if (vdev_writeable(vd)) { vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE, vd->vdev_trim_rate, vd->vdev_trim_partial, vd->vdev_trim_secure); @@ -1280,7 +1297,8 @@ vdev_autotrim_thread(void *arg) * Skip the metaslab when it has never been allocated * or when there are no recent frees to trim. */ - if (msp->ms_sm == NULL || + if (msp->ms_load_state == METASLAB_LOAD_UNLOADABLE || + msp->ms_sm == NULL || zfs_range_tree_is_empty(msp->ms_trim)) { mutex_exit(&msp->ms_lock); metaslab_enable(msp, B_FALSE, B_FALSE); @@ -1413,10 +1431,11 @@ vdev_autotrim_thread(void *arg) */ if (zfs_flags & ZFS_DEBUG_TRIM) { mutex_enter(&msp->ms_lock); - VERIFY0(metaslab_load(msp)); - VERIFY3P(tap[0].trim_msp, ==, msp); - zfs_range_tree_walk(trim_tree, - vdev_trim_range_verify, &tap[0]); + if (metaslab_load(msp) == 0) { + VERIFY3P(tap[0].trim_msp, ==, msp); + zfs_range_tree_walk(trim_tree, + vdev_trim_range_verify, &tap[0]); + } mutex_exit(&msp->ms_lock); } diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index 96e7f605229d..7d2321cc5220 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -392,7 +392,8 @@ tags = ['functional', 'cli_root', 'zfs_wait'] [tests/functional/cli_root/zhack] tests = ['zhack_label_repair_001', 'zhack_label_repair_002', - 'zhack_label_repair_003', 'zhack_label_repair_004', 'zhack_metaslab_leak'] + 'zhack_label_repair_003', 'zhack_label_repair_004', 'zhack_metaslab_leak', + 'zhack_metaslab_repair'] pre = post = tags = ['functional', 'cli_root', 'zhack'] diff --git a/tests/unit/.gitignore b/tests/unit/.gitignore index 3f240887d5cc..470234ffe934 100644 --- a/tests/unit/.gitignore +++ b/tests/unit/.gitignore @@ -5,4 +5,5 @@ /test_zap /test_namecheck /test_btree +/test_range_tree /test_sha2 diff --git a/tests/unit/Makefile.am b/tests/unit/Makefile.am index e1a44dcb295e..dc2508fbdb47 100644 --- a/tests/unit/Makefile.am +++ b/tests/unit/Makefile.am @@ -18,6 +18,7 @@ UNIT_TESTS = \ %D%/test_zap \ %D%/test_namecheck \ %D%/test_btree \ + %D%/test_range_tree \ %D%/test_fletcher \ %D%/test_sha2 noinst_PROGRAMS = $(UNIT_TESTS) @@ -69,6 +70,20 @@ nodist_%C%_test_btree_SOURCES = \ libunit.la +%C%_test_range_tree_CFLAGS = $(AM_CFLAGS) + +nodist_%C%_test_range_tree_SOURCES = \ + module/zfs/btree.c \ + module/zfs/range_tree.c + +%C%_test_range_tree_SOURCES = \ + %D%/test_range_tree.c + +%C%_test_range_tree_LDADD = \ + libspl.la \ + libunit.la + + %C%_test_fletcher_CFLAGS = $(AM_CFLAGS) nodist_%C%_test_fletcher_SOURCES = \ diff --git a/tests/unit/test_range_tree.c b/tests/unit/test_range_tree.c new file mode 100644 index 000000000000..aae9011edfac --- /dev/null +++ b/tests/unit/test_range_tree.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + */ + +/* + * Copyright (c) 2026, James Hilliard. + */ + +#include +#include + +#include "unit.h" + +static zfs_range_tree_t * +range_tree_create(void) +{ + return (zfs_range_tree_create(NULL, ZFS_RANGE_SEG64, NULL, 0, 0)); +} + +static void +range_tree_destroy(zfs_range_tree_t *rt) +{ + zfs_range_tree_vacate(rt, NULL, NULL); + zfs_range_tree_destroy(rt); +} + +static MunitResult +test_try_add(const MunitParameter params[], void *data) +{ + (void) params, (void) data; + + zfs_range_tree_t *rt = range_tree_create(); + + unit_true(zfs_range_tree_try_add(rt, 0x1000, 0x1000)); + unit_true(zfs_range_tree_try_add(rt, 0x3000, 0x1000)); + unit_true(zfs_range_tree_try_add(rt, 0x2000, 0x1000)); + unit_eq(zfs_range_tree_numsegs(rt), 1); + unit_eq(zfs_range_tree_space(rt), 0x3000); + unit_true(zfs_range_tree_contains(rt, 0x1000, 0x3000)); + + range_tree_destroy(rt); + return (MUNIT_OK); +} + +static MunitResult +test_try_add_overlap(const MunitParameter params[], void *data) +{ + (void) params, (void) data; + + zfs_range_tree_t *rt = range_tree_create(); + + zfs_range_tree_add(rt, 0x1000, 0x2000); + zfs_range_tree_add(rt, 0x5000, 0x1000); + + unit_false(zfs_range_tree_try_add(rt, 0x1000, 0x1000)); + unit_false(zfs_range_tree_try_add(rt, 0x2800, 0x1000)); + unit_eq(zfs_range_tree_numsegs(rt), 2); + unit_eq(zfs_range_tree_space(rt), 0x3000); + unit_true(zfs_range_tree_contains(rt, 0x1000, 0x2000)); + unit_true(zfs_range_tree_contains(rt, 0x5000, 0x1000)); + unit_false(zfs_range_tree_contains(rt, 0x3000, 0x2000)); + + range_tree_destroy(rt); + return (MUNIT_OK); +} + +static MunitResult +test_try_remove(const MunitParameter params[], void *data) +{ + (void) params, (void) data; + + zfs_range_tree_t *rt = range_tree_create(); + + zfs_range_tree_add(rt, 0x1000, 0x3000); + unit_true(zfs_range_tree_try_remove(rt, 0x2000, 0x1000)); + unit_eq(zfs_range_tree_numsegs(rt), 2); + unit_eq(zfs_range_tree_space(rt), 0x2000); + unit_true(zfs_range_tree_contains(rt, 0x1000, 0x1000)); + unit_true(zfs_range_tree_contains(rt, 0x3000, 0x1000)); + unit_false(zfs_range_tree_contains(rt, 0x2000, 0x1000)); + + unit_true(zfs_range_tree_try_remove(rt, 0x1000, 0x1000)); + unit_true(zfs_range_tree_try_remove(rt, 0x3000, 0x1000)); + unit_true(zfs_range_tree_is_empty(rt)); + + range_tree_destroy(rt); + return (MUNIT_OK); +} + +static MunitResult +test_try_remove_missing(const MunitParameter params[], void *data) +{ + (void) params, (void) data; + + zfs_range_tree_t *rt = range_tree_create(); + + zfs_range_tree_add(rt, 0x1000, 0x2000); + zfs_range_tree_add(rt, 0x5000, 0x1000); + + unit_false(zfs_range_tree_try_remove(rt, 0x3000, 0x1000)); + unit_false(zfs_range_tree_try_remove(rt, 0x2800, 0x3000)); + unit_false(zfs_range_tree_try_remove(rt, 0x1000, 0x4000)); + unit_eq(zfs_range_tree_numsegs(rt), 2); + unit_eq(zfs_range_tree_space(rt), 0x3000); + unit_true(zfs_range_tree_contains(rt, 0x1000, 0x2000)); + unit_true(zfs_range_tree_contains(rt, 0x5000, 0x1000)); + unit_false(zfs_range_tree_contains(rt, 0x3000, 0x2000)); + + range_tree_destroy(rt); + return (MUNIT_OK); +} + +static const MunitTest range_tree_tests[] = { + UNIT_TEST("try_add", test_try_add), + UNIT_TEST("try_add_overlap", test_try_add_overlap), + UNIT_TEST("try_remove", test_try_remove), + UNIT_TEST("try_remove_missing", test_try_remove_missing), + { 0 }, +}; + +static const MunitSuite range_tree_test_suite = { + "range_tree.", + range_tree_tests, + NULL, + 1, + MUNIT_SUITE_OPTION_NONE, +}; + +int +main(int argc, char **argv) +{ + int ret; + + zfs_btree_init(); + ret = munit_suite_main(&range_tree_test_suite, NULL, argc, argv); + zfs_btree_fini(); + return (ret); +} diff --git a/tests/zfs-tests/tests/Makefile.am b/tests/zfs-tests/tests/Makefile.am index c7bae4cb7ab0..d4453757d9f0 100644 --- a/tests/zfs-tests/tests/Makefile.am +++ b/tests/zfs-tests/tests/Makefile.am @@ -1084,6 +1084,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/cli_root/zhack/zhack_label_repair_003.ksh \ functional/cli_root/zhack/zhack_label_repair_004.ksh \ functional/cli_root/zhack/zhack_metaslab_leak.ksh \ + functional/cli_root/zhack/zhack_metaslab_repair.ksh \ functional/cli_root/zpool_add/add_nested_replacing_spare.ksh \ functional/cli_root/zpool_add/add-o_ashift.ksh \ functional/cli_root/zpool_add/add_prop_ashift.ksh \ diff --git a/tests/zfs-tests/tests/functional/cli_root/zhack/zhack_metaslab_repair.ksh b/tests/zfs-tests/tests/functional/cli_root/zhack/zhack_metaslab_repair.ksh new file mode 100755 index 000000000000..fc47b1cf59cc --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zhack/zhack_metaslab_repair.ksh @@ -0,0 +1,185 @@ +#!/bin/ksh +# SPDX-License-Identifier: CDDL-1.0 +# ZTS supplies STF_SUITE, TESTPOOL, and DISKS. +# shellcheck disable=SC2154 + +# +# Verify that repairable metaslab damage is conservatively replaced with a +# canonical space map and malformed maps remain unavailable. +# + +. "$STF_SUITE"/include/libtest.shlib + +verify_runnable "global" + +typeset old_debug_load +typeset old_keep_logs +typeset corruption +typeset tmpdir + +function cleanup +{ + log_must set_tunable64 METASLAB_DEBUG_LOAD "$old_debug_load" + log_must set_tunable64 KEEP_LOG_SPACEMAPS_AT_EXPORT "$old_keep_logs" + if poolexists "$TESTPOOL"; then + destroy_pool "$TESTPOOL" + fi + rm -rf "$tmpdir" +} + +function wait_for_repair +{ + typeset pool=$1 + typeset -i attempt + + for ((attempt = 0; attempt < 10; attempt++)); do + if zpool history -il "$pool" | grep -q "metaslab repair"; then + return 0 + fi + zpool sync "$pool" || return 1 + done + + zpool history -il "$pool" | grep -q "metaslab repair" +} + +function run_repair_test +{ + typeset corruption=$1 + typeset log_spacemap=$2 + typeset checksum + typeset repaired_checksum + typeset injection + typeset injection_log="$tmpdir/injection-$corruption-$log_spacemap" + typeset allocated_map="$tmpdir/map-$corruption-$log_spacemap" + typeset vdev + typeset offset + typeset size + typeset -a vdevs + + read -r -A vdevs <<< "$DISKS" + + log_note "testing $corruption corruption with log_spacemap " \ + "$log_spacemap" + log_must zpool create -o feature@log_spacemap="$log_spacemap" \ + "$TESTPOOL" "${vdevs[@]}" + log_must dd if=/dev/urandom of="/$TESTPOOL/data" bs=1M count=16 + checksum=$(cksum "/$TESTPOOL/data") + log_must zpool sync "$TESTPOOL" + log_must dd if=/dev/urandom of="/$TESTPOOL/churn" bs=128k count=10 + log_must zpool sync "$TESTPOOL" + + if [[ "$log_spacemap" == "enabled" ]]; then + log_must set_tunable64 KEEP_LOG_SPACEMAPS_AT_EXPORT 1 + log_must_busy zpool export "$TESTPOOL" + log_must eval "zhack -o zfs_keep_log_spacemaps_at_export=1 " \ + "metaslab corrupt $TESTPOOL $corruption > $injection_log" + else + log_must set_tunable64 KEEP_LOG_SPACEMAPS_AT_EXPORT 0 + log_must_busy zpool export "$TESTPOOL" + log_must eval "zhack metaslab corrupt $TESTPOOL $corruption " \ + "> $injection_log" + fi + + # Invalid summaries are scheduled for repair during metaslab setup. + if [[ "$corruption" == "invalid-summary" ]]; then + log_must set_tunable64 METASLAB_DEBUG_LOAD 0 + else + # Load every metaslab so import finds the injected overlap. + log_must set_tunable64 METASLAB_DEBUG_LOAD 1 + fi + log_must zpool import "$TESTPOOL" + + log_must wait_for_repair "$TESTPOOL" + + log_must set_tunable64 METASLAB_DEBUG_LOAD 0 + log_must set_tunable64 KEEP_LOG_SPACEMAPS_AT_EXPORT 0 + log_must_busy zpool export "$TESTPOOL" + + # zdb -b enables leak tracking, which strictly loads every space map. + log_must zdb -e -b "$TESTPOOL" + log_must eval "zdb -e -m --allocated-map $TESTPOOL > $allocated_map" + + if [[ "$corruption" != "invalid-summary" ]]; then + injection=$(<"$injection_log") + vdev=${injection#*vdev=} + vdev=${vdev%% *} + offset=${injection#*offset=} + offset=${offset%% *} + size=${injection#*size=} + size=${size%% *} + + # The complete ambiguous range must remain allocated. + log_must awk -v target_vdev="$vdev" \ + -v target_start="$offset" -v target_size="$size" " + \$1 == \"vdev\" { current_vdev = \$2 } + current_vdev == target_vdev && \$1 == \"ALLOC:\" && + \$2 <= target_start && + \$2 + \$3 >= target_start + target_size { found = 1 } + END { exit !found } + " "$allocated_map" + fi + + log_must zpool import "$TESTPOOL" + repaired_checksum=$(cksum "/$TESTPOOL/data") + log_must test "$repaired_checksum" = "$checksum" + log_must zpool destroy "$TESTPOOL" +} + +function run_unloadable_test +{ + typeset checksum + typeset reopened_checksum + typeset before + typeset after + typeset -a vdevs + + read -r -A vdevs <<< "$DISKS" + + log_note "testing an invalid metaslab space map entry" + log_must zpool create -o feature@log_spacemap=disabled \ + "$TESTPOOL" "${vdevs[@]}" + log_must dd if=/dev/urandom of="/$TESTPOOL/data" bs=1M count=16 + checksum=$(cksum "/$TESTPOOL/data") + log_must zpool sync "$TESTPOOL" + log_must_busy zpool export "$TESTPOOL" + + before=$(kstat metaslab_stats.load_unloadable) + log_must zhack metaslab corrupt "$TESTPOOL" invalid-entry + log_must set_tunable64 METASLAB_DEBUG_LOAD 1 + log_must zpool import "$TESTPOOL" + + after=$(kstat metaslab_stats.load_unloadable) + log_must test "$after" -gt "$before" + + # Waiting succeeds after the invalid metaslab suspends initialization. + log_must zpool initialize -w "$TESTPOOL" "${vdevs[0]}" + log_must eval "zpool status -i $TESTPOOL | grep -F ${vdevs[0]} | " \ + "grep -q suspended" + + log_must zpool initialize -c "$TESTPOOL" "${vdevs[0]}" + log_must_busy zpool export "$TESTPOOL" + log_must zpool import "$TESTPOOL" + + log_must set_tunable64 METASLAB_DEBUG_LOAD 0 + reopened_checksum=$(cksum "/$TESTPOOL/data") + log_must test "$reopened_checksum" = "$checksum" + log_must zpool destroy "$TESTPOOL" +} + +log_assert "damaged metaslab space maps are repaired or quarantined safely" + +old_debug_load=$(get_tunable METASLAB_DEBUG_LOAD) +old_keep_logs=$(get_tunable KEEP_LOG_SPACEMAPS_AT_EXPORT) +tmpdir=$(mktemp -d "$TEST_BASE_DIR/zhack_metaslab_repair.XXXXXX") || + log_fail "failed to create temporary directory" +log_onexit cleanup + +for corruption in duplicate-free partial-free duplicate-alloc partial-alloc \ + chained invalid-summary; do + run_repair_test "$corruption" enabled +done + +run_repair_test partial-free disabled +run_unloadable_test + +log_pass "metaslab damage was handled without changing referenced data"