zvol: return EBUSY when setting volmode/snapdev on in-use device

module/zfs/zvol.c

@@ -1805,6 +1805,63 @@ zvol_remove_minors_impl(zvol_task_t *task)
 	list_destroy(&remove_list);
 }
 
+/*
+ * Check if the named dataset or any of its descendant zvols are
+ * currently open (in use).  This is used to verify that it is safe
+ * to change a property that would affect the minor of the dataset
+ * itself and all inherited children.
+ */
+static boolean_t
+zvol_minors_in_use(const char *name)
+{
+	zvol_state_t *zv;
+	int namelen = strlen(name);
+
+	rw_enter(&zvol_state_lock, RW_READER);
+	for (zv = list_head(&zvol_state_list); zv != NULL;
+	    zv = list_next(&zvol_state_list, zv)) {
+		mutex_enter(&zv->zv_state_lock);
+		if (strncmp(zv->zv_name, name, namelen) == 0 &&
+		    (zv->zv_name[namelen] == '\0' ||
+		    zv->zv_name[namelen] == '/') &&
+		    zv->zv_open_count > 0) {
+			mutex_exit(&zv->zv_state_lock);
+			rw_exit(&zvol_state_lock);
+			return (B_TRUE);
+		}
+		mutex_exit(&zv->zv_state_lock);
+	}
+	rw_exit(&zvol_state_lock);
+	return (B_FALSE);
+}
+
+/*
+ * Check if any snapshot zvol minor under the given dataset is in use.
+ * Matches names of the form "name@snapname" where "name" is the prefix.
+ */
+static boolean_t
+zvol_snapshot_minors_in_use(const char *name)
+{
+	zvol_state_t *zv;
+	int namelen = strlen(name);
+
+	rw_enter(&zvol_state_lock, RW_READER);
+	for (zv = list_head(&zvol_state_list); zv != NULL;
+	    zv = list_next(&zvol_state_list, zv)) {
+		mutex_enter(&zv->zv_state_lock);
+		if (strncmp(zv->zv_name, name, namelen) == 0 &&
+		    zv->zv_name[namelen] == '@' &&
+		    zv->zv_open_count > 0) {
+			mutex_exit(&zv->zv_state_lock);
+			rw_exit(&zvol_state_lock);
+			return (B_TRUE);
+		}
+		mutex_exit(&zv->zv_state_lock);
+	}
+	rw_exit(&zvol_state_lock);
+	return (B_FALSE);
+}
+
 /* Remove minor for this specific volume only */
 static int
 zvol_remove_minor_impl(const char *name)
@@ -1939,9 +1996,11 @@ zvol_set_volmode_impl(zvol_task_t *task)
 		return;
 	if (zv != NULL) {
 		old_volmode = zv->zv_volmode;
-		mutex_exit(&zv->zv_state_lock);
-		if (old_volmode == volmode)
+		if (old_volmode == volmode) {
+			mutex_exit(&zv->zv_state_lock);
 			return;
+		}
+		mutex_exit(&zv->zv_state_lock);
 		zvol_wait_close(zv);
 	}
 	cookie = spl_fstrans_mark();
@@ -2129,6 +2188,25 @@ zvol_set_common(const char *ddname, zfs_prop_t prop, zprop_source_t source,
 	mutex_init(&zsda.zsda_lock, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&zsda.zsda_cv, NULL, CV_DEFAULT, NULL);
 
+	/*
+	 * Changing volmode or hiding snapdev on an open zvol would leave
+	 * the device in an inconsistent state: the property would be changed
+	 * on disk but the minor would not be removed.  Return EBUSY before
+	 * modifying the property so the caller knows to close the device
+	 * first.  This checks both the target dataset and any descendant
+	 * zvols that would inherit the change.
+	 */
+	if (prop == ZFS_PROP_VOLMODE &&
+	    zvol_minors_in_use(ddname)) {
+		error = SET_ERROR(EBUSY);
+		goto out;
+	}
+	if (prop == ZFS_PROP_SNAPDEV && val == ZFS_SNAPDEV_HIDDEN &&
+	    zvol_snapshot_minors_in_use(ddname)) {
+		error = SET_ERROR(EBUSY);
+		goto out;
+	}
+
 	error = spa_open(ddname, &spa, FTAG);
 	if (error != 0)
 		goto out;

tests/runfiles/linux.run

@@ -245,7 +245,7 @@ tests = ['groupspace_001_pos', 'groupspace_002_pos', 'groupspace_003_pos',
 tags = ['functional', 'userquota']
 
 [tests/functional/zvol/zvol_misc:Linux]
-tests = ['zvol_misc_fua']
+tests = ['zvol_misc_fua', 'zvol_misc_snapdev_inuse', 'zvol_misc_volmode_inuse']
 tags = ['functional', 'zvol', 'zvol_misc']
 
 [tests/functional/idmap_mount:Linux]

tests/zfs-tests/tests/Makefile.am

@@ -2482,8 +2482,10 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/zvol/zvol_misc/zvol_misc_hierarchy.ksh \
 	functional/zvol/zvol_misc/zvol_misc_rename_inuse.ksh \
 	functional/zvol/zvol_misc/zvol_misc_snapdev.ksh \
+	functional/zvol/zvol_misc/zvol_misc_snapdev_inuse.ksh \
 	functional/zvol/zvol_misc/zvol_misc_trim.ksh \
 	functional/zvol/zvol_misc/zvol_misc_volmode.ksh \
+	functional/zvol/zvol_misc/zvol_misc_volmode_inuse.ksh \
 	functional/zvol/zvol_misc/zvol_misc_zil.ksh \
 	functional/zvol/zvol_stress/cleanup.ksh \
 	functional/zvol/zvol_stress/setup.ksh \

tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_snapdev_inuse.ksh

@@ -0,0 +1,88 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions and
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2026 by Heonje LEE. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/zvol/zvol_common.shlib
+. $STF_SUITE/tests/functional/zvol/zvol_misc/zvol_misc_common.kshlib
+
+#
+# DESCRIPTION:
+# Verify 'zfs set snapdev=hidden' on a ZVOL whose snapshot device is in use
+# returns EBUSY instead of entering an uninterruptible D-state sleep.
+#
+# STRATEGY:
+# 1. Create a ZVOL with snapdev=visible and take a snapshot
+# 2. Open the snapshot block device to simulate in-use state
+#    (snapshot devices are read-only, so new_fs cannot be used)
+# 3. Verify setting snapdev=hidden fails (EBUSY) while device is open
+# 4. Release the device and verify snapdev=hidden succeeds
+#
+
+verify_runnable "global"
+
+HOLD_PID=""
+
+function cleanup
+{
+	[[ -n "$HOLD_PID" ]] && kill "$HOLD_PID" 2>/dev/null
+	[[ -n "$HOLD_PID" ]] && wait "$HOLD_PID" 2>/dev/null
+	datasetexists $ZVOL && destroy_dataset $ZVOL -r
+	is_linux && udev_cleanup
+}
+
+log_assert "Verify 'zfs set snapdev=hidden' on in-use ZVOL returns EBUSY"
+log_onexit cleanup
+
+ZVOL="$TESTPOOL/vol"
+SNAP="$ZVOL@snap"
+SNAPDEV="$ZVOL_DEVDIR/$SNAP"
+
+# 1. Create a ZVOL with snapdev=visible and take a snapshot
+log_must zfs create -V $VOLSIZE "$ZVOL"
+log_must zfs set snapdev=visible "$ZVOL"
+log_must zfs snapshot "$SNAP"
+blockdev_exists "$SNAPDEV"
+
+# 2. Open the snapshot block device to simulate in-use state.
+#    Snapshot devices are read-only, so new_fs (mkfs.ext4) cannot be used;
+#    instead, hold the device open with a background process.
+sleep 600 < "$SNAPDEV" &
+HOLD_PID=$!
+
+# 3. Verify setting snapdev=hidden fails while device is open
+is_linux && udev_wait
+log_mustnot zfs set snapdev=hidden "$ZVOL"
+
+# 4. Release the device and verify snapdev=hidden succeeds
+kill "$HOLD_PID" 2>/dev/null
+wait "$HOLD_PID" 2>/dev/null
+HOLD_PID=""
+is_linux && udev_wait
+log_must zfs set snapdev=hidden "$ZVOL"
+blockdev_missing "$SNAPDEV"
+
+log_pass "Setting snapdev=hidden on in-use snapshot ZVOL returns EBUSY"

tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_volmode_inuse.ksh

@@ -0,0 +1,80 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions and
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2026 by Heonje LEE. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/zvol/zvol_common.shlib
+. $STF_SUITE/tests/functional/zvol/zvol_misc/zvol_misc_common.kshlib
+
+#
+# DESCRIPTION:
+# Verify 'zfs set volmode' on a ZVOL whose block device is in use returns
+# EBUSY instead of entering an uninterruptible D-state sleep.
+#
+# STRATEGY:
+# 1. Create a ZVOL, format it, and mount the block device
+# 2. Verify changing volmode fails (EBUSY) while the device is mounted
+# 3. Unmount and verify volmode can be changed successfully
+#
+
+verify_runnable "global"
+
+function cleanup
+{
+	ismounted "$MNTPFS" $NEWFS_DEFAULT_FS && log_must umount "$MNTPFS"
+	[[ -d "$MNTPFS" ]] && log_must rmdir "$MNTPFS"
+	datasetexists $ZVOL && destroy_dataset $ZVOL -r
+	is_linux && udev_cleanup
+}
+
+log_assert "Verify 'zfs set volmode' on in-use ZVOL returns EBUSY"
+log_onexit cleanup
+
+ZVOL="$TESTPOOL/vol"
+ZDEV="$ZVOL_DEVDIR/$ZVOL"
+MNTPFS="$TESTDIR/zvol_inuse_volmode"
+
+# 1. Create a ZVOL, format it, and mount the block device
+log_must zfs create -V $VOLSIZE "$ZVOL"
+block_device_wait "$ZDEV"
+log_must eval "new_fs $ZDEV >/dev/null 2>&1"
+log_must mkdir "$MNTPFS"
+log_must mount "$ZDEV" "$MNTPFS"
+
+# 2. Verify changing volmode fails while the device is mounted
+is_linux && udev_wait
+log_mustnot zfs set volmode=none "$ZVOL"
+is_linux && udev_wait
+log_mustnot zfs set volmode=dev "$ZVOL"
+
+# 3. Unmount and verify volmode can be changed successfully
+log_must umount "$MNTPFS"
+log_must rmdir "$MNTPFS"
+is_linux && udev_wait
+log_must zfs set volmode=none "$ZVOL"
+blockdev_missing $ZDEV
+
+log_pass "Setting volmode on in-use ZVOL correctly returns EBUSY"