metaslab: Repair overlapping space map entries automatically

cmd/zhack.c

@@ -37,6 +37,7 @@
 #include <ctype.h>
 #include <sys/stat.h>
 #include <sys/zfs_context.h>
+#include <sys/zfs_debug.h>
 #include <sys/spa.h>
 #include <sys/spa_impl.h>
 #include <sys/dmu.h>
@@ -112,7 +113,11 @@ usage(void)
 	    "    <device> : path to vdev\n"
 	    "\n"
 	    "    metaslab leak <pool>\n"
-	    "        apply allocation map from zdb to specified pool\n");
+	    "        apply allocation map from zdb to specified pool\n"
+	    "    metaslab corrupt <pool> <type>\n"
+	    "        inject duplicate-free, partial-free, duplicate-alloc,\n"
+	    "        partial-alloc, chained, invalid-summary, or\n"
+	    "        invalid-entry damage\n");
 	exit(1);
 }
 
@@ -785,6 +790,241 @@ zhack_do_metaslab_leak(int argc, char **argv)
 	spa_close(spa, FTAG);
 }
 
+typedef enum zhack_metaslab_corruption {
+	ZHACK_METASLAB_DUPLICATE_FREE,
+	ZHACK_METASLAB_PARTIAL_FREE,
+	ZHACK_METASLAB_DUPLICATE_ALLOC,
+	ZHACK_METASLAB_PARTIAL_ALLOC,
+	ZHACK_METASLAB_CHAINED,
+	ZHACK_METASLAB_INVALID_SUMMARY,
+	ZHACK_METASLAB_INVALID_ENTRY
+} zhack_metaslab_corruption_t;
+
+typedef struct zhack_metaslab_corrupt_arg {
+	zhack_metaslab_corruption_t zmca_type;
+	const char *zmca_name;
+	boolean_t zmca_injected;
+	uint64_t zmca_vdev;
+	uint64_t zmca_metaslab;
+	uint64_t zmca_offset;
+	uint64_t zmca_size;
+} zhack_metaslab_corrupt_arg_t;
+
+static boolean_t
+zhack_metaslab_corrupt_range(metaslab_t *msp,
+    zhack_metaslab_corruption_t type, uint64_t *offset, uint64_t *size)
+{
+	zfs_range_tree_t *rt = msp->ms_allocatable;
+	uint64_t unit = 1ULL << msp->ms_sm->sm_shift;
+	uint64_t cursor = msp->ms_start;
+	zfs_btree_index_t where;
+	zfs_range_seg_t *rs;
+
+	for (rs = zfs_btree_first(&rt->rt_root, &where); rs != NULL;
+	    rs = zfs_btree_next(&rt->rt_root, &where, &where)) {
+		uint64_t start = zfs_rs_get_start(rs, rt);
+		uint64_t end = zfs_rs_get_end(rs, rt);
+
+		if ((type == ZHACK_METASLAB_DUPLICATE_ALLOC) &&
+		    start - cursor >= unit) {
+			*offset = cursor;
+			*size = unit;
+			return (B_TRUE);
+		}
+		if (type == ZHACK_METASLAB_DUPLICATE_FREE &&
+		    end - start >= unit) {
+			*offset = start;
+			*size = unit;
+			return (B_TRUE);
+		}
+		if ((type == ZHACK_METASLAB_PARTIAL_FREE ||
+		    type == ZHACK_METASLAB_PARTIAL_ALLOC) &&
+		    start - msp->ms_start >= unit) {
+			*offset = start - unit;
+			*size = 2 * unit;
+			return (B_TRUE);
+		}
+		if (type == ZHACK_METASLAB_CHAINED && end - start >= 2 * unit) {
+			*offset = start;
+			*size = 2 * unit;
+			return (B_TRUE);
+		}
+		if ((type == ZHACK_METASLAB_PARTIAL_FREE ||
+		    type == ZHACK_METASLAB_PARTIAL_ALLOC) &&
+		    msp->ms_start + msp->ms_size - end >= unit) {
+			*offset = end - unit;
+			*size = 2 * unit;
+			return (B_TRUE);
+		}
+		cursor = end;
+	}
+
+	if (type == ZHACK_METASLAB_DUPLICATE_ALLOC &&
+	    msp->ms_start + msp->ms_size - cursor >= unit) {
+		*offset = cursor;
+		*size = unit;
+		return (B_TRUE);
+	}
+
+	return (B_FALSE);
+}
+
+static void
+zhack_metaslab_corrupt_invalid_entry(space_map_t *sm, dmu_tx_t *tx)
+{
+	uint64_t entry = SM_OFFSET_ENCODE(sm->sm_size >> sm->sm_shift) |
+	    SM_TYPE_ENCODE(SM_ALLOC) | SM_RUN_ENCODE(1);
+
+	dmu_buf_will_dirty(sm->sm_dbuf, tx);
+	dmu_write(sm->sm_os, space_map_object(sm), space_map_length(sm),
+	    sizeof (entry), &entry, tx, DMU_READ_NO_PREFETCH);
+	sm->sm_phys->smp_length += sizeof (entry);
+}
+
+static void
+zhack_metaslab_corrupt_sync(void *arg, dmu_tx_t *tx)
+{
+	zhack_metaslab_corrupt_arg_t *zmca = arg;
+	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
+	vdev_t *rvd = spa->spa_root_vdev;
+
+	for (uint64_t c = 0; c < rvd->vdev_children; c++) {
+		vdev_t *vd = rvd->vdev_child[c];
+		if (vd->vdev_mg == NULL)
+			continue;
+
+		for (uint64_t m = 0; m < vd->vdev_ms_count; m++) {
+			metaslab_t *msp = vd->vdev_ms[m];
+
+			mutex_enter(&msp->ms_lock);
+			if (msp->ms_sm == NULL || metaslab_load(msp) != 0) {
+				mutex_exit(&msp->ms_lock);
+				continue;
+			}
+
+			uint64_t offset = 0;
+			uint64_t size = 0;
+			boolean_t invalid_summary = zmca->zmca_type ==
+			    ZHACK_METASLAB_INVALID_SUMMARY;
+			boolean_t invalid_entry = zmca->zmca_type ==
+			    ZHACK_METASLAB_INVALID_ENTRY;
+			if (invalid_summary) {
+				dmu_buf_will_dirty(msp->ms_sm->sm_dbuf, tx);
+				msp->ms_sm->sm_phys->smp_alloc =
+				    -(int64_t)(1ULL << msp->ms_sm->sm_shift);
+			} else if (invalid_entry) {
+				offset = msp->ms_start + msp->ms_size;
+				size = 1ULL << msp->ms_sm->sm_shift;
+			} else if (!zhack_metaslab_corrupt_range(msp,
+			    zmca->zmca_type, &offset, &size)) {
+				mutex_exit(&msp->ms_lock);
+				continue;
+			}
+
+			space_map_t *sm = msp->ms_sm;
+			zfs_range_tree_t *rt = NULL;
+			if (!invalid_summary && !invalid_entry) {
+				rt = zfs_range_tree_create(NULL,
+				    ZFS_RANGE_SEG64, NULL, 0, 0);
+				uint64_t first_size =
+				    zmca->zmca_type == ZHACK_METASLAB_CHAINED ?
+				    size / 2 : size;
+				zfs_range_tree_add(rt, offset, first_size);
+			}
+			mutex_exit(&msp->ms_lock);
+
+			if (invalid_entry) {
+				zhack_metaslab_corrupt_invalid_entry(sm, tx);
+			} else if (rt != NULL) {
+				maptype_t maptype =
+				    zmca->zmca_type ==
+				    ZHACK_METASLAB_DUPLICATE_ALLOC ||
+				    zmca->zmca_type ==
+				    ZHACK_METASLAB_PARTIAL_ALLOC ?
+				    SM_ALLOC : SM_FREE;
+				space_map_write(sm, rt, maptype,
+				    SM_NO_VDEVID, tx);
+				if (zmca->zmca_type == ZHACK_METASLAB_CHAINED) {
+					zfs_range_tree_vacate(rt, NULL, NULL);
+					zfs_range_tree_add(rt, offset, size);
+					space_map_write(sm, rt, SM_ALLOC,
+					    SM_NO_VDEVID, tx);
+				}
+				zfs_range_tree_vacate(rt, NULL, NULL);
+				zfs_range_tree_destroy(rt);
+			}
+
+			zmca->zmca_injected = B_TRUE;
+			zmca->zmca_vdev = vd->vdev_id;
+			zmca->zmca_metaslab = msp->ms_id;
+			zmca->zmca_offset = offset;
+			zmca->zmca_size = size;
+			return;
+		}
+	}
+}
+
+static void
+zhack_do_metaslab_corrupt(int argc, char **argv)
+{
+	char *target;
+	spa_t *spa;
+
+	argc--;
+	argv++;
+	if (argc != 2) {
+		(void) fprintf(stderr,
+		    "error: expected a pool name and corruption type\n");
+		usage();
+	}
+	target = argv[0];
+
+	zhack_metaslab_corrupt_arg_t zmca = {
+		.zmca_name = argv[1],
+	};
+	if (strcmp(zmca.zmca_name, "duplicate-free") == 0) {
+		zmca.zmca_type = ZHACK_METASLAB_DUPLICATE_FREE;
+	} else if (strcmp(zmca.zmca_name, "partial-free") == 0) {
+		zmca.zmca_type = ZHACK_METASLAB_PARTIAL_FREE;
+	} else if (strcmp(zmca.zmca_name, "duplicate-alloc") == 0) {
+		zmca.zmca_type = ZHACK_METASLAB_DUPLICATE_ALLOC;
+	} else if (strcmp(zmca.zmca_name, "partial-alloc") == 0) {
+		zmca.zmca_type = ZHACK_METASLAB_PARTIAL_ALLOC;
+	} else if (strcmp(zmca.zmca_name, "chained") == 0) {
+		zmca.zmca_type = ZHACK_METASLAB_CHAINED;
+	} else if (strcmp(zmca.zmca_name, "invalid-summary") == 0) {
+		zmca.zmca_type = ZHACK_METASLAB_INVALID_SUMMARY;
+	} else if (strcmp(zmca.zmca_name, "invalid-entry") == 0) {
+		zmca.zmca_type = ZHACK_METASLAB_INVALID_ENTRY;
+	} else {
+		(void) fprintf(stderr, "error: unknown corruption type: %s\n",
+		    zmca.zmca_name);
+		usage();
+	}
+
+	/*
+	 * Committing an intentionally inconsistent space map can make its
+	 * allocation summary disagree with the loaded metaslab.  Leave other
+	 * debug checks enabled while allowing the injector to close the pool.
+	 */
+	zfs_flags &= ~ZFS_DEBUG_METASLAB_VERIFY;
+	zhack_spa_open(target, B_FALSE, FTAG, &spa);
+
+	VERIFY0(dsl_sync_task(spa_name(spa), NULL,
+	    zhack_metaslab_corrupt_sync, &zmca, 5,
+	    ZFS_SPACE_CHECK_NONE));
+	if (!zmca.zmca_injected) {
+		fatal(spa, FTAG, "no suitable metaslab range available in '%s'",
+		    target);
+	}
+
+	(void) printf("injected %s: vdev=%" PRIu64
+	    " metaslab=%" PRIu64 " offset=%" PRIu64 " size=%" PRIu64 "\n",
+	    zmca.zmca_name, zmca.zmca_vdev, zmca.zmca_metaslab,
+	    zmca.zmca_offset, zmca.zmca_size);
+	spa_close(spa, FTAG);
+}
+
 static int
 zhack_do_metaslab(int argc, char **argv)
 {
@@ -801,6 +1041,8 @@ zhack_do_metaslab(int argc, char **argv)
 	subcommand = argv[0];
 	if (strcmp(subcommand, "leak") == 0) {
 		zhack_do_metaslab_leak(argc, argv);
+	} else if (strcmp(subcommand, "corrupt") == 0) {
+		zhack_do_metaslab_corrupt(argc, argv);
 	} else {
 		(void) fprintf(stderr, "error: unknown subcommand: %s\n",
 		    subcommand);

include/sys/metaslab_impl.h

@@ -338,6 +338,19 @@ struct metaslab_group {
  * ensure that allocations are not performed on the metaslab that is
  * being written.
  */
+typedef enum metaslab_load_state {
+	METASLAB_LOAD_NORMAL = 0,
+	METASLAB_LOAD_DEFERRED,
+	METASLAB_LOAD_UNLOADABLE,
+} metaslab_load_state_t;
+
+typedef struct metaslab_repair {
+	int64_t		msr_smp_alloc;
+	uint64_t	msr_allocated;
+	uint64_t	msr_entries;
+	uint64_t	msr_bytes;
+} metaslab_repair_t;
+
 struct metaslab {
 	/*
 	 * This is the main lock of the metaslab and its purpose is to
@@ -404,6 +417,12 @@ struct metaslab {
 	boolean_t	ms_condensing;	/* condensing? */
 	boolean_t	ms_condense_wanted;
 
+	/* Details retained until a repaired space map is condensed. */
+	metaslab_repair_t	ms_repair;
+	/* The on-disk smp_alloc is outside the metaslab's valid range. */
+	boolean_t	ms_smp_alloc_invalid;
+	metaslab_load_state_t	ms_load_state;
+
 	/*
 	 * The number of consumers which have disabled the metaslab.
 	 */

include/sys/range_tree.h

@@ -310,7 +310,17 @@ uint64_t zfs_range_tree_max(zfs_range_tree_t *rt);
 uint64_t zfs_range_tree_span(zfs_range_tree_t *rt);
 
 void zfs_range_tree_add(void *arg, uint64_t start, uint64_t size);
+/*
+ * For gap-free trees, try_add() succeeds only when the requested range does
+ * not overlap the tree, and try_remove() succeeds only when one segment
+ * contains the entire range. Both return B_FALSE without modifying the tree
+ * otherwise.
+ */
+boolean_t zfs_range_tree_try_add(zfs_range_tree_t *rt, uint64_t start,
+    uint64_t size);
 void zfs_range_tree_remove(void *arg, uint64_t start, uint64_t size);
+boolean_t zfs_range_tree_try_remove(zfs_range_tree_t *rt, uint64_t start,
+    uint64_t size);
 void zfs_range_tree_remove_fill(zfs_range_tree_t *rt, uint64_t start,
     uint64_t size);
 void zfs_range_tree_adjust_fill(zfs_range_tree_t *rt, zfs_range_seg_t *rs,

include/sys/space_map.h

@@ -208,9 +208,22 @@ boolean_t sm_entry_is_double_word(uint64_t e);
 
 typedef int (*sm_cb_t)(space_map_entry_t *sme, void *arg);
 
+typedef struct space_map_load_result {
+	/* Allocated bytes reconstructed from the entries that were loaded. */
+	uint64_t smlr_allocated;
+	uint64_t smlr_repaired_entries;
+	uint64_t smlr_affected_bytes;
+	space_map_entry_t smlr_first_entry;
+	space_map_entry_t smlr_last_entry;
+	uint64_t smlr_first_free_bytes;
+	uint64_t smlr_last_free_bytes;
+} space_map_load_result_t;
+
 int space_map_load(space_map_t *sm, zfs_range_tree_t *rt, maptype_t maptype);
 int space_map_load_length(space_map_t *sm, zfs_range_tree_t *rt,
     maptype_t maptype, uint64_t length);
+int space_map_load_length_repair(space_map_t *sm, zfs_range_tree_t *rt,
+    uint64_t length, space_map_load_result_t *result);
 int space_map_iterate(space_map_t *sm, uint64_t length,
     sm_cb_t callback, void *arg);
 int space_map_incremental_destroy(space_map_t *sm, sm_cb_t callback, void *arg,