diff --git a/config/kernel-ki_complete.m4 b/config/kernel-ki_complete.m4 new file mode 100644 index 000000000000..1110cd20d31c --- /dev/null +++ b/config/kernel-ki_complete.m4 @@ -0,0 +1,26 @@ +dnl # SPDX-License-Identifier: CDDL-1.0 +dnl # +dnl # 5.19 API change, +dnl # kiocb->ki_complete() reduced from 3 args to 2: +dnl # old: void (*ki_complete)(struct kiocb *, long, long) +dnl # new: void (*ki_complete)(struct kiocb *, long) +dnl # +AC_DEFUN([ZFS_AC_KERNEL_SRC_KIOCB_KI_COMPLETE], [ + ZFS_LINUX_TEST_SRC([kiocb_ki_complete_2args], [ + #include + ],[ + struct kiocb *kiocb = NULL; + kiocb->ki_complete(kiocb, 0); + ]) +]) + +AC_DEFUN([ZFS_AC_KERNEL_KIOCB_KI_COMPLETE], [ + AC_MSG_CHECKING([whether kiocb->ki_complete() wants 2 args]) + ZFS_LINUX_TEST_RESULT([kiocb_ki_complete_2args], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_2ARGS_KI_COMPLETE, 1, + [kiocb->ki_complete() wants 2 args]) + ],[ + AC_MSG_RESULT(no) + ]) +]) diff --git a/config/kernel.m4 b/config/kernel.m4 index 083b1476e81d..ebdb3ef70a70 100644 --- a/config/kernel.m4 +++ b/config/kernel.m4 @@ -90,6 +90,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [ ZFS_AC_KERNEL_SRC_VFS_REMAP_FILE_RANGE ZFS_AC_KERNEL_SRC_VFS_CLONE_FILE_RANGE ZFS_AC_KERNEL_SRC_VFS_DEDUPE_FILE_RANGE + ZFS_AC_KERNEL_SRC_KIOCB_KI_COMPLETE ZFS_AC_KERNEL_SRC_KMAP_ATOMIC_ARGS ZFS_AC_KERNEL_SRC_KMAP_LOCAL_PAGE ZFS_AC_KERNEL_SRC_FOLLOW_DOWN_ONE @@ -204,6 +205,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [ ZFS_AC_KERNEL_VFS_REMAP_FILE_RANGE ZFS_AC_KERNEL_VFS_CLONE_FILE_RANGE ZFS_AC_KERNEL_VFS_DEDUPE_FILE_RANGE + ZFS_AC_KERNEL_KIOCB_KI_COMPLETE ZFS_AC_KERNEL_KMAP_ATOMIC_ARGS ZFS_AC_KERNEL_KMAP_LOCAL_PAGE ZFS_AC_KERNEL_FOLLOW_DOWN_ONE diff --git a/include/os/linux/Makefile.am b/include/os/linux/Makefile.am index 6e01ccb4095e..e7d325ce36f7 100644 --- a/include/os/linux/Makefile.am +++ b/include/os/linux/Makefile.am @@ -24,6 +24,7 @@ kernel_sysdir = $(kerneldir)/sys kernel_sys_HEADERS = \ %D%/zfs/sys/abd_os.h \ %D%/zfs/sys/abd_impl_os.h \ + %D%/zfs/sys/dmu_direct_os.h \ %D%/zfs/sys/policy.h \ %D%/zfs/sys/trace_acl.h \ %D%/zfs/sys/trace_arc.h \ diff --git a/include/os/linux/spl/sys/uio.h b/include/os/linux/spl/sys/uio.h index 26c2c387caa3..4121f303e406 100644 --- a/include/os/linux/spl/sys/uio.h +++ b/include/os/linux/spl/sys/uio.h @@ -189,6 +189,22 @@ zfs_uio_iov_iter_init(zfs_uio_t *uio, struct iov_iter *iter, offset_t offset, memset(&uio->uio_dio, 0, sizeof (zfs_uio_dio_t)); } +/* + * Rewind an iter-backed uio's underlying iov_iter by nbytes. zfs_uiomove() + * advances the VFS iov_iter that uio_iter points at; a caller that saves and + * restores the zfs_uio_t struct on error (e.g. zfs_write_async()) restores + * only the scalar resid/offset, not the iterator position, since both structs + * share one iov_iter. Use this to undo the advance before handing the + * iterator to a fallback path. Only valid for UIO_ITER uios. + */ +static inline void +zfs_uio_iov_iter_revert(zfs_uio_t *uio, size_t nbytes) +{ + ASSERT3S(uio->uio_segflg, ==, UIO_ITER); + if (nbytes > 0) + iov_iter_revert(uio->uio_iter, nbytes); +} + #if defined(HAVE_ITER_IOV) #define zfs_uio_iter_iov(iter) iter_iov((iter)) #else diff --git a/include/os/linux/zfs/sys/dmu_direct_os.h b/include/os/linux/zfs/sys/dmu_direct_os.h new file mode 100644 index 000000000000..819632cb3daa --- /dev/null +++ b/include/os/linux/zfs/sys/dmu_direct_os.h @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or https://opensource.org/licenses/CDDL-1.0. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * Copyright 2026, tiehexue . All rights reserved. + * + */ + +#ifndef _SYS_DMU_OS_H +#define _SYS_DMU_OS_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Async Direct I/O completion callback type (shared by read and write). + */ +typedef void (dmu_abd_done_func_t)(void *arg, int error); + +/* + * Async Direct I/O read. Submits reads via the ZIO pipeline and returns + * immediately. The completion callback fires from ZIO taskq context when + * all reads finish. Caller retains ownership of 'data' until callback. + */ +int dmu_read_abd_async(dnode_t *dn, uint64_t offset, uint64_t size, + abd_t *data, dmu_flags_t flags, + dmu_abd_done_func_t *done, void *done_arg); + +/* + * Async Direct I/O write. Submits writes via the ZIO pipeline and returns + * immediately. The completion callback fires from ZIO taskq context when + * all writes finish. Caller retains ownership of 'data' until callback + * and must commit the transaction (tx) from the callback. + */ +int dmu_write_abd_async(dnode_t *dn, uint64_t offset, uint64_t size, + abd_t *data, dmu_flags_t flags, dmu_tx_t *tx, + dmu_abd_done_func_t *done, void *done_arg); + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_DMU_OS_H */ diff --git a/include/os/linux/zfs/sys/zfs_vfsops_os.h b/include/os/linux/zfs/sys/zfs_vfsops_os.h index 2058ef322945..2864fb554043 100644 --- a/include/os/linux/zfs/sys/zfs_vfsops_os.h +++ b/include/os/linux/zfs/sys/zfs_vfsops_os.h @@ -103,6 +103,9 @@ struct zfsvfs { boolean_t z_use_hold; /* held via dmu_objset_hold */ rrmlock_t z_teardown_lock; krwlock_t z_teardown_inactive_lock; + kmutex_t z_async_dio_lock; /* protects z_async_dio_inflight */ + kcondvar_t z_async_dio_cv; /* signals drain to teardown */ + uint64_t z_async_dio_inflight; /* submitted, not yet completed */ list_t z_all_znodes; /* all znodes in the fs */ unsigned long z_rollback_time; /* last online rollback time */ unsigned long z_snap_defer_time; /* last snapshot unmount deferral */ diff --git a/include/os/linux/zfs/sys/zfs_vnops_os.h b/include/os/linux/zfs/sys/zfs_vnops_os.h index 37574d42f477..4f03ac65ed45 100644 --- a/include/os/linux/zfs/sys/zfs_vnops_os.h +++ b/include/os/linux/zfs/sys/zfs_vnops_os.h @@ -94,6 +94,14 @@ extern int zfs_rename_idmap(znode_t *sdzp, char *snm, znode_t *tdzp, zidmap_t *idmap); extern int zfs_symlink_idmap(znode_t *dzp, char *name, vattr_t *vap, char *link, znode_t **zpp, cred_t *cr, int flags, zidmap_t *idmap); +/* async Direct I/O */ +struct kiocb; +extern int zfs_read_async(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr, + struct kiocb *kiocb); +extern int zfs_write_async(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr, + struct kiocb *kiocb); +extern void zfs_async_dio_init(void); +extern void zfs_async_dio_fini(void); #ifdef __cplusplus } diff --git a/include/sys/dmu_impl.h b/include/sys/dmu_impl.h index bae872bd1907..c549ff37de6d 100644 --- a/include/sys/dmu_impl.h +++ b/include/sys/dmu_impl.h @@ -271,6 +271,19 @@ int dmu_write_direct(zio_t *, dmu_buf_impl_t *, abd_t *, dmu_tx_t *); int dmu_read_abd(dnode_t *, uint64_t, uint64_t, abd_t *, dmu_flags_t); int dmu_write_abd(dnode_t *, uint64_t, uint64_t, abd_t *, dmu_flags_t, dmu_tx_t *); + +/* + * Shared helpers used by both synchronous and asynchronous DMU I/O paths. + * These dispatch per-dbuf ZIOs into a caller-provided root zio; the caller + * decides whether to zio_wait() (sync) or zio_nowait() + callback (async). + */ +int dmu_read_abd_dispatch(zio_t *rio, dnode_t *dn, uint64_t offset, + uint64_t size, abd_t *data, dmu_flags_t flags, + dmu_buf_t **dbp, int numbufs); +int dmu_write_abd_dispatch(zio_t *pio, dnode_t *dn, uint64_t offset, + uint64_t size, abd_t *data, dmu_flags_t flags, dmu_tx_t *tx, + dmu_buf_t **dbp, int numbufs); + #if defined(_KERNEL) int dmu_read_uio_direct(dnode_t *, zfs_uio_t *, uint64_t, dmu_flags_t); int dmu_write_uio_direct(dnode_t *, zfs_uio_t *, uint64_t, dmu_flags_t, diff --git a/include/sys/rrwlock.h b/include/sys/rrwlock.h index ad845e6dd5dc..b87371ca4cea 100644 --- a/include/sys/rrwlock.h +++ b/include/sys/rrwlock.h @@ -108,6 +108,7 @@ boolean_t rrm_held(rrmlock_t *rrl, krw_t rw); #define RRM_READ_HELD(x) rrm_held(x, RW_READER) #define RRM_WRITE_HELD(x) rrm_held(x, RW_WRITER) + #define RRM_LOCK_HELD(x) \ (rrm_held(x, RW_WRITER) || rrm_held(x, RW_READER)) diff --git a/include/sys/zfs_vnops.h b/include/sys/zfs_vnops.h index 08cf0e2a6e48..0781f9608924 100644 --- a/include/sys/zfs_vnops.h +++ b/include/sys/zfs_vnops.h @@ -31,9 +31,33 @@ extern int zfs_bclone_enabled; +/* + * Direct I/O tunables. zfs_dio_enabled can be set to 0 to force all + * I/O through the ARC; zfs_dio_strict returns EINVAL for unaligned + * DIO instead of falling back. + */ +extern int zfs_dio_enabled; +extern int zfs_dio_strict; + extern int zfs_fsync(znode_t *, int, cred_t *); extern int zfs_read(znode_t *, zfs_uio_t *, int, cred_t *); extern int zfs_write(znode_t *, zfs_uio_t *, int, cred_t *); + +/* + * Direct I/O page-pinning setup. Pins user pages for O_DIRECT reads, + * enforces alignment, and skips DIO for mmap'd or encrypted ranges. + * Returns 0 and sets UIO_DIRECT in uio->uio_extflg on success. + */ +extern int zfs_setup_direct(struct znode *, zfs_uio_t *, zfs_uio_rw_t, int *); + +/* + * Clear the SUID/SGID bits after a write by non-owner. + * Called from the async write completion path (zfs_vnops_os.c on Linux) + * as well as from the synchronous zfs_write(). + */ +extern void zfs_clear_setid_bits_if_necessary(zfsvfs_t *, znode_t *, cred_t *, + uint64_t *, dmu_tx_t *); + extern int zfs_holey(znode_t *, ulong_t, loff_t *); extern int zfs_access(znode_t *, int, int, cred_t *); extern int zfs_clone_range(znode_t *, uint64_t *, znode_t *, uint64_t *, diff --git a/module/Kbuild.in b/module/Kbuild.in index d6f8894f86ee..8a8774055350 100644 --- a/module/Kbuild.in +++ b/module/Kbuild.in @@ -458,6 +458,7 @@ ZFS_OBJS := \ ZFS_OBJS_OS := \ abd_os.o \ + dmu_direct_os.o \ arc_os.o \ kasan_compat.o \ mmp_os.o \ diff --git a/module/os/linux/zfs/dmu_direct_os.c b/module/os/linux/zfs/dmu_direct_os.c new file mode 100644 index 000000000000..bca8a2b0035b --- /dev/null +++ b/module/os/linux/zfs/dmu_direct_os.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or https://opensource.org/licenses/CDDL-1.0. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * Copyright 2026, tiehexue . All rights reserved. + * + */ + +/* + * Linux async Direct I/O variants for the DMU layer. + * + * dmu_read_abd_async() / dmu_write_abd_async() submit I/O via the ZIO + * pipeline and return immediately. Completions fire from ZIO taskq + * context via caller-provided callbacks (dmu_abd_done_func_t). + * + * These are Linux-only because only the Linux VFS layer has async + * kiocb / -EIOCBQUEUED infrastructure that benefits from non-blocking + * DMU entry points. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Shared async state for dmu_read_abd_async() and dmu_write_abd_async(). + */ +struct dmu_abd_async_state { + dmu_buf_t **ds_dbp; + int ds_numbufs; + dmu_abd_done_func_t *ds_done; + void *ds_done_arg; +}; + +static void +dmu_abd_async_done(zio_t *zio) +{ + struct dmu_abd_async_state *ds = zio->io_private; + int error = zio->io_error; + + dmu_buf_rele_array(ds->ds_dbp, ds->ds_numbufs, FTAG); + ds->ds_done(ds->ds_done_arg, error); + kmem_free(ds, sizeof (*ds)); +} + +/* + * Asynchronous variant of dmu_read_abd(). Uses the shared + * dmu_read_abd_dispatch() helper (common code) for the per-dbuf ZIO + * submission loop; only the async plumbing (root zio callback, state + * allocation, zio_nowait) lives here in the Linux-specific layer. + */ +int +dmu_read_abd_async(dnode_t *dn, uint64_t offset, uint64_t size, + abd_t *data, dmu_flags_t flags, + dmu_abd_done_func_t *done, void *done_arg) +{ + spa_t *spa = dn->dn_objset->os_spa; + dmu_buf_t **dbp; + int numbufs, err; + + ASSERT(flags & DMU_DIRECTIO); + ASSERT3P(done, !=, NULL); + + err = dmu_buf_hold_array_by_dnode(dn, offset, + size, B_FALSE, FTAG, &numbufs, &dbp, flags); + if (err) + return (err); + + struct dmu_abd_async_state *ds = + kmem_alloc(sizeof (*ds), KM_SLEEP); + ds->ds_dbp = dbp; + ds->ds_numbufs = numbufs; + ds->ds_done = done; + ds->ds_done_arg = done_arg; + + zio_t *rio = zio_root(spa, dmu_abd_async_done, ds, + ZIO_FLAG_CANFAIL); + + err = dmu_read_abd_dispatch(rio, dn, offset, size, data, flags, + dbp, numbufs); + + /* + * Dispatch the root zio. On error, dmu_read_abd_dispatch() has + * already set rio->io_error; the async done callback will release + * dbp and signal the caller. + */ + zio_nowait(rio); + return (0); +} + + + +/* + * Asynchronous variant of dmu_write_abd(). Uses the shared + * dmu_write_abd_dispatch() helper (common code) for the per-dbuf + * ZIO submission loop; only the async plumbing lives here. + */ +int +dmu_write_abd_async(dnode_t *dn, uint64_t offset, uint64_t size, + abd_t *data, dmu_flags_t flags, dmu_tx_t *tx, + dmu_abd_done_func_t *done, void *done_arg) +{ + spa_t *spa = dn->dn_objset->os_spa; + dmu_buf_t **dbp; + int numbufs, err; + + ASSERT(flags & DMU_DIRECTIO); + ASSERT3P(done, !=, NULL); + + err = dmu_buf_hold_array_by_dnode(dn, offset, + size, B_FALSE, FTAG, &numbufs, &dbp, flags); + if (err) + return (err); + + struct dmu_abd_async_state *ds = + kmem_alloc(sizeof (*ds), KM_SLEEP); + ds->ds_dbp = dbp; + ds->ds_numbufs = numbufs; + ds->ds_done = done; + ds->ds_done_arg = done_arg; + + zio_t *pio = zio_root(spa, dmu_abd_async_done, ds, + ZIO_FLAG_CANFAIL); + + err = dmu_write_abd_dispatch(pio, dn, offset, size, data, flags, tx, + dbp, numbufs); + + if (err != 0) + pio->io_error = err; + + zio_nowait(pio); + return (0); +} diff --git a/module/os/linux/zfs/zfs_ioctl_os.c b/module/os/linux/zfs/zfs_ioctl_os.c index ce6092be1da7..dc10a33df229 100644 --- a/module/os/linux/zfs/zfs_ioctl_os.c +++ b/module/os/linux/zfs/zfs_ioctl_os.c @@ -66,6 +66,7 @@ #include #include +#include #include #include @@ -302,6 +303,7 @@ openzfs_init_os(void) return (-error); } + zfs_async_dio_init(); zfs_sysfs_init(); printk(KERN_NOTICE "ZFS: Loaded module v%s-%s%s, " @@ -327,6 +329,7 @@ static void openzfs_fini_os(void) { zfs_sysfs_fini(); + zfs_async_dio_fini(); zfs_kmod_fini(); printk(KERN_NOTICE "ZFS: Unloaded module v%s-%s%s\n", diff --git a/module/os/linux/zfs/zfs_uio.c b/module/os/linux/zfs/zfs_uio.c index bfce9e6b5202..1eebb1607f78 100644 --- a/module/os/linux/zfs/zfs_uio.c +++ b/module/os/linux/zfs/zfs_uio.c @@ -506,6 +506,19 @@ zfs_uio_free_dio_pages(zfs_uio_t *uio, zfs_uio_rw_t rw) vmem_free(uio->uio_dio.pages, uio->uio_dio.npages * sizeof (struct page *)); + + /* + * Reset the Direct I/O state so the same uio can be re-setup by + * zfs_setup_direct(). The async read path pins pages and may then + * decline the request (EOPNOTSUPP) after freeing them, in which case + * the synchronous fallback re-runs zfs_setup_direct() on this uio. + * Stale npages/pages/pinned would make the re-pin write past the end + * of the newly allocated pages array and trip the npages ASSERT. + */ + uio->uio_dio.pages = NULL; + uio->uio_dio.npages = 0; + uio->uio_dio.pinned = B_FALSE; + uio->uio_extflg &= ~UIO_DIRECT; } #if defined(HAVE_PIN_USER_PAGES_UNLOCKED) @@ -666,6 +679,16 @@ zfs_uio_get_dio_pages_alloc(zfs_uio_t *uio, zfs_uio_rw_t rw) } vmem_free(uio->uio_dio.pages, size); + + /* + * Leave the uio in a clean state so a fallback path (e.g. the + * synchronous re-run after an async EOPNOTSUPP) can call + * zfs_setup_direct() again on the same uio. + */ + uio->uio_dio.pages = NULL; + uio->uio_dio.npages = 0; + uio->uio_dio.pinned = B_FALSE; + uio->uio_extflg &= ~UIO_DIRECT; return (error); } else { ASSERT3S(uio->uio_dio.npages, ==, npages); diff --git a/module/os/linux/zfs/zfs_vfsops.c b/module/os/linux/zfs/zfs_vfsops.c index acfe2a6cf8bf..7c5f015d0f6b 100644 --- a/module/os/linux/zfs/zfs_vfsops.c +++ b/module/os/linux/zfs/zfs_vfsops.c @@ -687,6 +687,9 @@ zfsvfs_create_impl(zfsvfs_t **zfvp, zfsvfs_t *zfsvfs, objset_t *os) list_create(&zfsvfs->z_all_znodes, sizeof (znode_t), offsetof(znode_t, z_link_node)); ZFS_TEARDOWN_INIT(zfsvfs); + mutex_init(&zfsvfs->z_async_dio_lock, NULL, MUTEX_DEFAULT, NULL); + cv_init(&zfsvfs->z_async_dio_cv, NULL, CV_DEFAULT, NULL); + zfsvfs->z_async_dio_inflight = 0; rw_init(&zfsvfs->z_teardown_inactive_lock, NULL, RW_DEFAULT, NULL); rw_init(&zfsvfs->z_fuid_lock, NULL, RW_DEFAULT, NULL); @@ -833,6 +836,9 @@ zfsvfs_free(zfsvfs_t *zfsvfs) mutex_destroy(&zfsvfs->z_lock); list_destroy(&zfsvfs->z_all_znodes); ZFS_TEARDOWN_DESTROY(zfsvfs); + ASSERT0(zfsvfs->z_async_dio_inflight); + mutex_destroy(&zfsvfs->z_async_dio_lock); + cv_destroy(&zfsvfs->z_async_dio_cv); rw_destroy(&zfsvfs->z_teardown_inactive_lock); rw_destroy(&zfsvfs->z_fuid_lock); for (i = 0; i != size; i++) { @@ -1236,6 +1242,21 @@ zfsvfs_teardown(zfsvfs_t *zfsvfs, boolean_t unmounting) ZFS_TEARDOWN_ENTER_WRITE(zfsvfs, FTAG); + /* + * Async DIO (zfs_read_async/zfs_write_async) holds the teardown + * lock only across submission; each submitted operation instead + * holds z_async_dio_inflight until its completion (ZIO/system + * taskq context) has made its last touch of the zfsvfs. Now that + * we hold the teardown lock as writer no new async I/O can be + * submitted; wait for the in-flight ones to finish before tearing + * anything down. Completions never take the teardown lock, so + * this cannot deadlock. + */ + mutex_enter(&zfsvfs->z_async_dio_lock); + while (zfsvfs->z_async_dio_inflight != 0) + cv_wait(&zfsvfs->z_async_dio_cv, &zfsvfs->z_async_dio_lock); + mutex_exit(&zfsvfs->z_async_dio_lock); + if (!unmounting) { /* * We purge the parent filesystem's super block as the diff --git a/module/os/linux/zfs/zfs_vnops_os.c b/module/os/linux/zfs/zfs_vnops_os.c index b867e20f071c..56e100bd7c2c 100644 --- a/module/os/linux/zfs/zfs_vnops_os.c +++ b/module/os/linux/zfs/zfs_vnops_os.c @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -71,6 +72,7 @@ #include #include #include +#include #include /* @@ -4515,3 +4517,828 @@ EXPORT_SYMBOL(zfs_map); module_param(zfs_delete_blocks, ulong, 0644); MODULE_PARM_DESC(zfs_delete_blocks, "Delete files larger than N blocks async"); #endif + +/* + * ========================================================================= + * Async Direct I/O + * ========================================================================= + * + * Submits O_DIRECT reads and writes to the ZIO pipeline and returns + * -EIOCBQUEUED to the Linux VFS so the submitting kworker can service + * other io_uring / libaio requests. Completion is signalled via + * kiocb->ki_complete() from ZIO taskq context. + * + * Reads: pin user pages into an ABD → dmu_read_abd_async(). + * The ABD is freed in the callback after DMA completes. + * + * Writes: copy user data into a kernel-linear ABD (avoids FOLL_LONGTERM + * pinning issues on RHEL/mainline ≥ 6.0) → dmu_write_abd_async(). + * On ZIO completion, metadata updates and transaction commit run + * on system_taskq in process context. + */ + +/* + * Dedicated caches for async read/write callback structs. + * Eliminates per-I/O kmalloc fast-path overhead and improves + * CPU cache locality by grouping same-type objects together. + */ +static kmem_cache_t *zfs_async_read_cb_cache = NULL; +static kmem_cache_t *zfs_async_write_cb_cache = NULL; + +/* + * Completion callback for zfs_read_async(). Invoked from ZIO taskq + * context when all blocks have been read. Cleans up the range lock, + * DIO pages, atime, and signals completion via kiocb->ki_complete(). + */ +struct zfs_async_read_cb +{ + struct kiocb *kiocb; + znode_t *zp; + zfsvfs_t *zfsvfs; + zfs_locked_range_t *lr; + zfs_uio_t uio; + ssize_t start_resid; + dmu_flags_t dflags; /* DMU flags from the original read */ + abd_t *data; /* ABD wrapping user pages; freed in cb */ + boolean_t is_retry; + abd_t *user_data; /* saved user-pages ABD during retry */ +}; + +/* + * Async DIO in-flight accounting. The submitting thread holds the + * teardown lock (zfs_enter) only across submission; completions run in + * ZIO/system taskq threads, which must NOT release a teardown reader + * lock acquired by another thread: once a teardown writer is waiting + * (rr_writer_wanted -- umount, export, rollback, recv -F), rrwlock + * tracks new readers on the submitting thread's rrn list, and a + * cross-thread rrw_exit() misses that node and corrupts the anon/linked + * reader counts (refcount VERIFY panic, or the teardown writer waits + * forever). Instead, each submitted async operation holds + * z_async_dio_inflight until its completion has made its last touch of + * the zfsvfs/znode, and zfsvfs_teardown() drains the counter right + * after taking the teardown lock as writer -- at which point no new + * async I/O can be submitted. + */ +static void +zfs_async_dio_hold(zfsvfs_t *zfsvfs) +{ + mutex_enter(&zfsvfs->z_async_dio_lock); + zfsvfs->z_async_dio_inflight++; + mutex_exit(&zfsvfs->z_async_dio_lock); +} + +static void +zfs_async_dio_rele(zfsvfs_t *zfsvfs) +{ + mutex_enter(&zfsvfs->z_async_dio_lock); + ASSERT3U(zfsvfs->z_async_dio_inflight, >, 0); + if (--zfsvfs->z_async_dio_inflight == 0) + cv_broadcast(&zfsvfs->z_async_dio_cv); + mutex_exit(&zfsvfs->z_async_dio_lock); +} + +static void +zfs_async_read_complete(void *arg, int error) +{ + struct zfs_async_read_cb *cb = arg; + ssize_t read; + + /* + * A Direct I/O read that fails checksum verification is suspect: + * a concurrent writer may have modified the buffer in flight, + * refer to PR#18844. On first ECKSUM, re-read into a temporary + * ABD via an async chained dmu_read_abd_async(). The temporary + * ABD lives in stable kernel memory where no concurrent writer can + * overwrite it, so the checksum verify on the retry passes for a + * benign race and fails only for genuine on-disk corruption. + * + * If the retry succeeds, copy the stable data into the pinned user + * pages and signal success. If the retry also fails, or if + * submission of the retry fails, signal EIO. + */ + if (error == ECKSUM) { + if (!cb->is_retry) { + abd_t *tmp = abd_alloc_for_io(cb->start_resid, B_FALSE); + + cb->user_data = cb->data; + cb->data = tmp; + cb->is_retry = B_TRUE; + + dmu_buf_t *db = sa_get_db(cb->zp->z_sa_hdl); + int retry_err = dmu_read_abd_async( + DB_DNODE((dmu_buf_impl_t *)db), + zfs_uio_offset(&cb->uio), + cb->start_resid, + tmp, + cb->dflags, + zfs_async_read_complete, cb); + + if (retry_err != 0) { + /* + * Retry submission failed: roll back to the + * state before the retry and signal EIO. + */ + abd_free(tmp); + cb->data = cb->user_data; + cb->user_data = NULL; + cb->is_retry = B_FALSE; + error = SET_ERROR(EIO); + goto out; + } + + return; + } + + /* Retry also got ECKSUM: genuine on-disk corruption. */ + error = SET_ERROR(EIO); + } + + /* + * On return from a retry, regardless of error, free the temporary + * kernel ABD and restore cb->data to the user-pages ABD so the + * final cleanup below frees the correct ABD. + */ + if (cb->is_retry) { + if (error == 0) { + /* + * Retry succeeded: copy stable data from the + * temporary ABD into the pinned user pages. + */ + abd_copy(cb->user_data, cb->data, cb->start_resid); + } + abd_free(cb->data); + cb->data = cb->user_data; + cb->user_data = NULL; + } + + /* + * For the DIO (ABD) path, dmu_read_abd_async() writes data directly + * to the user pages via DMA, so uio_resid is never decremented. + * On success, report the full requested size as the bytes read. + */ + if (error == 0) + read = cb->start_resid; + else + read = 0; + +out: + + dataset_kstats_update_read_kstats(&cb->zfsvfs->z_kstat, read); + zfs_rangelock_exit(cb->lr); + + /* + * Unpin DIO pages — DMA is complete (or failed). + */ + if (cb->uio.uio_extflg & UIO_DIRECT) + zfs_uio_free_dio_pages(&cb->uio, UIO_READ); + + ZFS_ACCESSTIME_STAMP(cb->zfsvfs, cb->zp); + + /* + * Last touch of the zfsvfs/znode: drop the in-flight hold that + * has kept zfsvfs_teardown() at bay since submission (the + * teardown reader lock itself was released by the submitting + * thread; see zfs_async_dio_hold()). + */ + zfs_async_dio_rele(cb->zfsvfs); + + /* + * Free the ABD that wraps the user pages. The data is now in the + * user pages, and the pages are unpinned by zfs_uio_free_dio_pages() + * above. In the synchronous path, the caller frees the ABD after + * zio_wait() returns; here we must do it in the callback. + */ + if (cb->data != NULL) + abd_free(cb->data); + +#ifdef HAVE_2ARGS_KI_COMPLETE + cb->kiocb->ki_complete(cb->kiocb, error ? -error : read); +#else + cb->kiocb->ki_complete(cb->kiocb, error ? -error : read, 0); +#endif + kmem_cache_free(zfs_async_read_cb_cache, cb); +} + +/* + * Async Direct I/O read. Pins user pages (requires current->mm), + * submits reads via dmu_read_abd_async(), and returns 0 on success. + * The caller should then return -EIOCBQUEUED to the VFS. + * + * On I/O completion, zfs_async_read_complete() handles cleanup and + * calls kiocb->ki_complete(). + * + * Returns 0 on successful async submission, or positive errno if the + * read must be done synchronously. + */ +int +zfs_read_async(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr, + struct kiocb *kiocb) +{ + (void) cr; + int error; + ssize_t n; + + zfsvfs_t *zfsvfs = ZTOZSB(zp); + if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0) + return (error); + + if (zp->z_pflags & ZFS_AV_QUARANTINED) { + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EACCES)); + } + if (Z_ISDIR(ZTOTYPE(zp))) { + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EISDIR)); + } + + /* Pin user pages for DIO — requires current->mm (we're in kworker) */ + error = zfs_setup_direct(zp, uio, UIO_READ, &ioflag); + if (error || !(uio->uio_extflg & UIO_DIRECT)) { + zfs_exit(zfsvfs, FTAG); + return (error ? error : SET_ERROR(EOPNOTSUPP)); + } + + zfs_locked_range_t *lr = zfs_rangelock_enter(&zp->z_rangelock, + zfs_uio_offset(uio), zfs_uio_resid(uio), RL_READER); + + if (zfs_uio_offset(uio) >= zp->z_size) { + /* + * File truncated between caller's i_size_read() guard + * and our range lock. No data to read and no ZIO + * queued, so return an error — the caller falls + * through to sync rather than returning -EIOCBQUEUED. + */ + zfs_rangelock_exit(lr); + zfs_uio_free_dio_pages(uio, UIO_READ); + ZFS_ACCESSTIME_STAMP(zfsvfs, zp); + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EOPNOTSUPP)); + } + + n = MIN(zfs_uio_resid(uio), zp->z_size - zfs_uio_offset(uio)); + ssize_t aligned_n = P2ALIGN_TYPED(n, PAGE_SIZE, ssize_t); + if (aligned_n == 0) { + zfs_rangelock_exit(lr); + zfs_uio_free_dio_pages(uio, UIO_READ); + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EOPNOTSUPP)); + } + + /* + * Larger reads fall back to the synchronous path, which chunks them. + */ + if (aligned_n > SPA_MAXBLOCKSIZE) { + zfs_rangelock_exit(lr); + zfs_uio_free_dio_pages(uio, UIO_READ); + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EOPNOTSUPP)); + } + + /* Build ABD from pinned pages */ + offset_t offset = zfs_uio_offset(uio); + offset_t page_idx = (offset - zfs_uio_soffset(uio)) >> PAGESHIFT; + ASSERT3U(page_idx, <, uio->uio_dio.npages); + abd_t *data = abd_alloc_from_pages(&uio->uio_dio.pages[page_idx], + offset & (PAGESIZE - 1), aligned_n); + + dmu_flags_t dflags = DMU_READ_PREFETCH | DMU_DIRECTIO; + if (ioflag & O_DIRECT) + dflags |= DMU_UNCACHEDIO; + + struct zfs_async_read_cb *cb = + kmem_cache_alloc(zfs_async_read_cb_cache, KM_SLEEP); + cb->kiocb = kiocb; + cb->zp = zp; + cb->zfsvfs = zfsvfs; + cb->lr = lr; + cb->uio = *uio; + cb->uio.uio_resid = aligned_n; + cb->start_resid = aligned_n; + cb->dflags = dflags; + cb->data = data; + cb->is_retry = B_FALSE; + cb->user_data = NULL; + + /* + * Held until the completion's last touch of the zfsvfs; taken + * before submission because the completion may fire (and rele) + * before dmu_read_abd_async() returns. A nonzero return means + * the callback will never run, so the error path must rele. + */ + zfs_async_dio_hold(zfsvfs); + + dmu_buf_t *db = sa_get_db(zp->z_sa_hdl); + error = dmu_read_abd_async(DB_DNODE((dmu_buf_impl_t *)db), + offset, aligned_n, data, dflags, zfs_async_read_complete, cb); + + if (error) { + zfs_async_dio_rele(zfsvfs); + abd_free(data); + zfs_rangelock_exit(lr); + zfs_uio_free_dio_pages(uio, UIO_READ); + zfs_exit(zfsvfs, FTAG); + kmem_cache_free(zfs_async_read_cb_cache, cb); + return (error); + } + + /* + * The in-flight hold now stands in for the teardown reader lock, + * which must be released here, on the thread that acquired it. + */ + zfs_exit(zfsvfs, FTAG); + return (0); +} + +/* + * Callback state for zfs_write_async(). Carries everything needed to + * complete the write once the ZIOs finish. + */ +struct zfs_async_write_cb { + struct kiocb *kiocb; + znode_t *zp; + zfsvfs_t *zfsvfs; + zfs_locked_range_t *lr; + zfs_uio_t uio; + ssize_t start_resid; + abd_t *data; + dmu_tx_t *tx; + offset_t woff; + ssize_t tx_bytes; + sa_bulk_attr_t bulk[5]; + int bulk_count; + uint64_t mtime[2]; + uint64_t ctime[2]; + uint64_t clear_setid_bits_txg; + cred_t *cr; + boolean_t do_commit; + int error; + ssize_t wrote; +}; + +static void +zfs_async_write_task(void *arg) +{ + struct zfs_async_write_cb *cb = arg; + zilog_t *zilog = cb->zfsvfs->z_log; + + if (cb->error == 0) { + if (cb->cr != NULL) { + zfs_clear_setid_bits_if_necessary(cb->zfsvfs, cb->zp, + cb->cr, &cb->clear_setid_bits_txg, cb->tx); + } + + zfs_tstamp_update_setup(cb->zp, CONTENT_MODIFIED, + cb->mtime, cb->ctime); + + uint64_t end_size; + while ((end_size = cb->zp->z_size) < + (uint64_t)(cb->woff + cb->wrote)) + (void) atomic_cas_64(&cb->zp->z_size, end_size, + (uint64_t)(cb->woff + cb->wrote)); + + if (cb->zp->z_is_sa) + cb->zp->z_has_seq = B_TRUE; + + /* + * A failed sa_bulk_update() means the metadata (size, + * timestamps, flags) did not fully land even though the data + * blocks were written. Report the error to the caller so it + * can retry; the tx is still committed below, matching the + * synchronous path. + */ + int sa_err = sa_bulk_update(cb->zp->z_sa_hdl, cb->bulk, + cb->bulk_count, cb->tx); + if (sa_err != 0 && cb->error == 0) + cb->error = sa_err; + + zfs_log_write(zilog, cb->tx, TX_WRITE, cb->zp, cb->woff, + cb->wrote, cb->do_commit, B_TRUE, NULL, NULL); + + dmu_tx_commit(cb->tx); + } else { + /* + * The tx was assigned at submit time and must be committed + * even though the write failed: dmu_tx_abort() VERIFYs + * tx_txg == 0 and panics on an assigned tx. The + * synchronous path likewise commits on a post-assign write + * error (see zfs_write()); committing an empty metadata + * transaction is legal, so the size/timestamp updates and + * zfs_log_write() above are skipped. + */ + dmu_tx_commit(cb->tx); + cb->wrote = 0; + } + + dataset_kstats_update_write_kstats(&cb->zfsvfs->z_kstat, cb->wrote); + zfs_znode_update_vfs(cb->zp); + zfs_rangelock_exit(cb->lr); + if (cb->uio.uio_extflg & UIO_DIRECT) + zfs_uio_free_dio_pages(&cb->uio, UIO_WRITE); + + /* + * zil_commit before teardown lock release. This matches zfs_write(), + * which also commits the log whenever bytes were written, even if a + * metadata update reported an error. On a ZIO failure cb->wrote is 0, + * so nothing is flushed. + */ + if (cb->wrote > 0 && cb->do_commit) { + int zil_err = zil_commit(zilog, cb->zp->z_id); + if (zil_err != 0 && cb->error == 0) + cb->error = zil_err; + } + + /* + * Last touch of the zfsvfs/znode: drop the in-flight hold that + * has kept zfsvfs_teardown() at bay since submission (the + * teardown reader lock itself was released by the submitting + * thread; see zfs_async_dio_hold()). Must come after the + * zil_commit above so teardown's zil_close() cannot race it. + */ + zfs_async_dio_rele(cb->zfsvfs); + + if (cb->data != NULL) + abd_free(cb->data); + if (cb->cr != NULL) + crfree(cb->cr); + +#ifdef HAVE_2ARGS_KI_COMPLETE + cb->kiocb->ki_complete(cb->kiocb, + cb->error ? -cb->error : cb->wrote); +#else + cb->kiocb->ki_complete(cb->kiocb, + cb->error ? -cb->error : cb->wrote, 0); +#endif + kmem_cache_free(zfs_async_write_cb_cache, cb); +} + +/* + * Completion callback for dmu_write_abd_async(). Called from ZIO + * taskq context when all writes finish. Dispatches to system_taskq + * for metadata operations, transaction commit, cleanup, and ki_complete + * — all in process context, not ZIO taskq. + */ +static void +zfs_async_write_complete(void *arg, int error) +{ + struct zfs_async_write_cb *cb = arg; + + cb->wrote = cb->start_resid - cb->uio.uio_resid; + cb->error = error; + + /* + * TQ_SLEEP dispatch sleeps for memory rather than failing + * (task_alloc() falls through to kmem_alloc(KM_SLEEP)); the only + * TASKQID_INVALID return is from a taskq being destroyed, which + * cannot happen to system_taskq while this module is loaded. + * Do not fall back to running the task inline: this is ZIO + * completion context, where zil_commit() can self-deadlock, and + * clobbering the error code of a successful write would report + * failure for data that is already on disk. + */ + VERIFY3U(taskq_dispatch(system_taskq, zfs_async_write_task, + cb, TQ_SLEEP), !=, TASKQID_INVALID); +} + +/* + * Async Direct I/O write. Copies user data into a kernel ABD, acquires + * locks, creates a transaction, and dispatches writes via + * dmu_write_abd_async(). The caller returns -EIOCBQUEUED to the VFS. + * + * On I/O completion, zfs_async_write_complete() fires from ZIO taskq + * and dispatches to system_taskq for metadata operations and cleanup + * in process context. + */ +int +zfs_write_async(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr, + struct kiocb *kiocb) +{ + int error; + ssize_t n; + + zfsvfs_t *zfsvfs = ZTOZSB(zp); + if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0) + return (error); + + if (zp->z_pflags & ZFS_AV_QUARANTINED) { + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EACCES)); + } + if (Z_ISDIR(ZTOTYPE(zp))) { + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EISDIR)); + } + if (zfs_is_readonly(zfsvfs)) { + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EROFS)); + } + if ((zp->z_pflags & ZFS_IMMUTABLE) || + ((zp->z_pflags & ZFS_APPENDONLY) && !(ioflag & O_APPEND) && + (zfs_uio_offset(uio) < zp->z_size))) { + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EPERM)); + } + + offset_t woff = ioflag & O_APPEND ? + zp->z_size : zfs_uio_offset(uio); + if (woff < 0) { + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EINVAL)); + } + + /* + * Async Direct I/O writes copy user data into a kernel ABD + * instead of pinning user pages via pin_user_pages_unlocked(). + * On RHEL kernels (and mainline >= 6.0), all pin_user_pages*() + * variants implicitly add FOLL_LONGTERM which fails with ENOMEM + * under concurrent I/O (iodepth=64 → 2048 concurrently held + * pages). Copying is safe on all kernel versions and the memcpy + * cost is negligible compared to disk I/O latency. + */ + n = zfs_uio_resid(uio); + if (n == 0) { + zfs_exit(zfsvfs, FTAG); + return (0); + } + + n = P2ALIGN_TYPED(n, PAGE_SIZE, ssize_t); + if (n == 0) { + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EOPNOTSUPP)); + } + + /* + * Replicate DIO eligibility checks from zfs_setup_direct(). + * Any condition that would skip DIO in the sync path returns + * EOPNOTSUPP so the caller falls back to zfs_write(). The + * sync path will then re-check everything including zfs_dio_strict. + */ + if (zfsvfs->z_os->os_direct == ZFS_DIRECT_ALWAYS) + ioflag |= O_DIRECT; + + if (!zfs_dio_enabled || + zfsvfs->z_os->os_direct == ZFS_DIRECT_DISABLED) { + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EOPNOTSUPP)); + } + + if (!zfs_uio_page_aligned(uio) || + !zfs_uio_aligned(uio, PAGE_SIZE)) { + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EOPNOTSUPP)); + } + + if (n < zp->z_blksz || + zn_has_cached_data(zp, zfs_uio_offset(uio), + zfs_uio_offset(uio) + n - 1)) { + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EOPNOTSUPP)); + } + + /* + * Larger requests fall back to the synchronous path, which + * chunks them per transaction. + */ + if (n > SPA_MAXBLOCKSIZE) { + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EOPNOTSUPP)); + } + + /* + * Copy the user data into a kernel ABD BEFORE taking the range lock + * to avoid deadlock. + * + * Save the uio before the copy: every error path from here on (the + * copy itself, or a rejection once the range lock is held) must + * rewind the shared iov_iter and restore the uio so the synchronous + * fallback in zpl_iter_write() retries the full write. + */ + abd_t *data = abd_alloc_linear(n, B_FALSE); + zfs_uio_t saved_uio = *uio; + + error = zfs_uiomove(abd_to_buf(data), n, UIO_WRITE, uio); + if (error) { + /* + * The copy faulted partway: zfs_uiomove() advanced the + * underlying iov_iter by the bytes consumed. Rewind it + * before restoring the saved uio. + */ + zfs_uio_iov_iter_revert(uio, + saved_uio.uio_resid - uio->uio_resid); + *uio = saved_uio; + abd_free(data); + zfs_exit(zfsvfs, FTAG); + return (error); + } + + zfs_locked_range_t *lr; + if (ioflag & O_APPEND) { + lr = zfs_rangelock_enter(&zp->z_rangelock, 0, n, RL_APPEND); + woff = lr->lr_offset; + if (lr->lr_length == UINT64_MAX) + woff = zp->z_size; + zfs_uio_setoffset(uio, woff); + } else { + lr = zfs_rangelock_enter(&zp->z_rangelock, woff, n, RL_WRITER); + } + + /* + * The write offset is now fixed by the range lock. Every rejection + * below must restore the uio (the copy above has consumed the user + * data) and free the ABD before returning. + */ + if (zn_rlimit_fsize_uio(zp, &saved_uio)) { + zfs_rangelock_exit(lr); + zfs_uio_iov_iter_revert(uio, + saved_uio.uio_resid - uio->uio_resid); + *uio = saved_uio; + abd_free(data); + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EFBIG)); + } + + if (lr->lr_length == UINT64_MAX) { + zfs_rangelock_exit(lr); + zfs_uio_iov_iter_revert(uio, + saved_uio.uio_resid - uio->uio_resid); + *uio = saved_uio; + abd_free(data); + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EOPNOTSUPP)); + } + + if (woff >= MAXOFFSET_T) { + zfs_rangelock_exit(lr); + zfs_uio_iov_iter_revert(uio, + saved_uio.uio_resid - uio->uio_resid); + *uio = saved_uio; + abd_free(data); + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EFBIG)); + } + if ((uint64_t)n > MAXOFFSET_T - woff) + n = MAXOFFSET_T - woff; + n = P2ALIGN_TYPED(n, PAGE_SIZE, ssize_t); + if (n == 0) { + zfs_rangelock_exit(lr); + zfs_uio_iov_iter_revert(uio, + saved_uio.uio_resid - uio->uio_resid); + *uio = saved_uio; + abd_free(data); + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EOPNOTSUPP)); + } + + /* + * Direct I/O writes are dispatched per-dbuf by + * dmu_write_abd_dispatch(), which slices the source ABD at + * (db_offset - offset) assuming every covered dbuf lies fully + * within [offset, offset+n). That only holds for a write that + * is block-aligned in both offset and length. The synchronous + * path enforces this in dmu_write_uio_dnode() via + * zfs_dio_aligned() and routes any unaligned span through the + * ARC. Mirror that gate here: for a block-misaligned write, + * fall back to the sync path (EOPNOTSUPP) rather than dispatch + * a slice whose per-dbuf offset math underflows and trips the + * VERIFY in abd_get_offset_size(). + */ + if (!zfs_dio_aligned(woff, n, zp->z_blksz)) { + zfs_rangelock_exit(lr); + zfs_uio_iov_iter_revert(uio, + saved_uio.uio_resid - uio->uio_resid); + *uio = saved_uio; + abd_free(data); + zfs_exit(zfsvfs, FTAG); + return (SET_ERROR(EOPNOTSUPP)); + } + + boolean_t do_commit = (ioflag & (O_SYNC | O_DSYNC)) || + (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS); + dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, ZFS_SEQ_MAY_GROW(zp)); + dmu_buf_impl_t *db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl); + DB_DNODE_ENTER(db); + dmu_tx_hold_write_by_dnode(tx, DB_DNODE(db), woff, n); + DB_DNODE_EXIT(db); + zfs_sa_upgrade_txholds(tx, zp); + error = dmu_tx_assign(tx, DMU_TX_WAIT); + if (error) { + dmu_tx_abort(tx); + abd_free(data); + zfs_rangelock_exit(lr); + zfs_uio_iov_iter_revert(uio, + saved_uio.uio_resid - uio->uio_resid); + *uio = saved_uio; + zfs_exit(zfsvfs, FTAG); + return (error); + } + + dmu_flags_t dflags = DMU_DIRECTIO; + if (ioflag & O_DIRECT) + dflags |= DMU_UNCACHEDIO; + + struct zfs_async_write_cb *cb = + kmem_cache_alloc(zfs_async_write_cb_cache, KM_SLEEP); + cb->kiocb = kiocb; + cb->zp = zp; + cb->zfsvfs = zfsvfs; + cb->lr = lr; + cb->uio = *uio; + cb->uio.uio_resid = 0; /* all data already copied to ABD */ + cb->start_resid = n; + cb->data = data; + cb->tx = tx; + cb->woff = woff; + cb->tx_bytes = n; + cb->cr = cr; + crhold(cr); + + cb->bulk_count = 0; + SA_ADD_BULK_ATTR(cb->bulk, cb->bulk_count, + SA_ZPL_MTIME(zfsvfs), NULL, &cb->mtime, 16); + SA_ADD_BULK_ATTR(cb->bulk, cb->bulk_count, + SA_ZPL_CTIME(zfsvfs), NULL, &cb->ctime, 16); + SA_ADD_BULK_ATTR(cb->bulk, cb->bulk_count, + SA_ZPL_SIZE(zfsvfs), NULL, &zp->z_size, 8); + SA_ADD_BULK_ATTR(cb->bulk, cb->bulk_count, + SA_ZPL_FLAGS(zfsvfs), NULL, &zp->z_pflags, 8); + if (zp->z_is_sa) + SA_ADD_BULK_ATTR(cb->bulk, cb->bulk_count, + SA_ZPL_SEQ(zfsvfs), NULL, &zp->z_seq, 8); + + cb->clear_setid_bits_txg = 0; + cb->do_commit = do_commit; + cb->error = 0; + cb->wrote = 0; + + /* + * Held until the completion's last touch of the zfsvfs; taken + * before submission because the completion may fire (and rele) + * before dmu_write_abd_async() returns. A nonzero return means + * the callback will never run, so the error path must rele. + */ + zfs_async_dio_hold(zfsvfs); + + error = dmu_write_abd_async(DB_DNODE(db), woff, n, data, dflags, + tx, zfs_async_write_complete, cb); + if (error) { + /* + * Restore the uio so the sync fallback in zpl_iter_write() + * retries the full write. zfs_uiomove() already consumed + * the user data into the ABD, so rewind the shared iov_iter + * before the struct restore; otherwise the fallback sees an + * exhausted iterator and fails a write whose data was fine. + * The tx is assigned -- commit, don't abort (abort VERIFYs + * tx_txg == 0). + */ + zfs_async_dio_rele(zfsvfs); + zfs_uio_iov_iter_revert(uio, + saved_uio.uio_resid - uio->uio_resid); + *uio = saved_uio; + crfree(cr); + abd_free(data); + zfs_rangelock_exit(lr); + dmu_tx_commit(tx); + zfs_exit(zfsvfs, FTAG); + kmem_cache_free(zfs_async_write_cb_cache, cb); + return (error); + } + + /* + * The in-flight hold now stands in for the teardown reader lock, + * which must be released here, on the thread that acquired it. + */ + zfs_exit(zfsvfs, FTAG); + return (0); +} + +/* + * Module init: create dedicated caches for async I/O callbacks. + * Called once from zfs_kmod_init() — no lazy-init races. + */ +void +zfs_async_dio_init(void) +{ + zfs_async_read_cb_cache = kmem_cache_create("zfs_async_read_cb", + sizeof (struct zfs_async_read_cb), 0, + NULL, NULL, NULL, NULL, NULL, 0); + zfs_async_write_cb_cache = kmem_cache_create("zfs_async_write_cb", + sizeof (struct zfs_async_write_cb), 0, + NULL, NULL, NULL, NULL, NULL, 0); +} + +/* + * Module fini: destroy caches created by zfs_async_dio_init(). + * Called once from zfs_kmod_fini(). + */ +void +zfs_async_dio_fini(void) +{ + if (zfs_async_read_cb_cache != NULL) { + kmem_cache_destroy(zfs_async_read_cb_cache); + zfs_async_read_cb_cache = NULL; + } + if (zfs_async_write_cb_cache != NULL) { + kmem_cache_destroy(zfs_async_write_cb_cache); + zfs_async_write_cb_cache = NULL; + } +} diff --git a/module/os/linux/zfs/zpl_file.c b/module/os/linux/zfs/zpl_file.c index 81e52c0610c1..34eac39ec9c8 100644 --- a/module/os/linux/zfs/zpl_file.c +++ b/module/os/linux/zfs/zpl_file.c @@ -211,6 +211,23 @@ zpl_file_accessed(struct file *filp) } } +/* + * Module parameter to enable/disable async Direct I/O (reads and writes). + * When enabled, O_DIRECT operations on async kiocbs (libaio/io_uring) use + * zfs_read_async()/zfs_write_async(), which submit I/O to the ZIO pipeline + * and return -EIOCBQUEUED to the VFS. Completion is signalled via + * kiocb->ki_complete() from ZIO/system taskq context. + * + * Default: 0 (disabled) + */ +static unsigned int zfs_async_dio_enabled = 0; + +#ifdef CONFIG_SYSFS +module_param(zfs_async_dio_enabled, uint, 0644); +MODULE_PARM_DESC(zfs_async_dio_enabled, + "Enable async Direct I/O via -EIOCBQUEUED (reads and writes)"); +#endif + static ssize_t zpl_iter_read(struct kiocb *kiocb, struct iov_iter *to) { @@ -222,6 +239,40 @@ zpl_iter_read(struct kiocb *kiocb, struct iov_iter *to) zfs_uio_iov_iter_init(&uio, to, kiocb->ki_pos, count); + /* + * If async DIO is enabled, the file is opened O_DIRECT, and + * this is an async kiocb, dispatch the read via zfs_read_async(). + * On success we return -EIOCBQUEUED; on error fall through to + * the synchronous path. Without O_DIRECT, reads go through the + * ARC — the data copy is synchronous and the async path adds no + * benefit. + * + * Skip async dispatch when there is nothing to read: count == 0 + * or the file position is already at/past EOF. Both are handled + * correctly by the synchronous zfs_read() below. + */ + if (zfs_async_dio_enabled && !is_sync_kiocb(kiocb) && + count > 0 && + kiocb->ki_pos < i_size_read(filp->f_mapping->host) && + ((filp->f_flags & O_DIRECT) || + ITOZSB(filp->f_mapping->host)->z_os->os_direct == + ZFS_DIRECT_ALWAYS)) { + crhold(cr); + cookie = spl_fstrans_mark(); + + ssize_t ret = -zfs_read_async( + ITOZ(filp->f_mapping->host), &uio, + filp->f_flags | zfs_io_flags(kiocb), cr, kiocb); + + spl_fstrans_unmark(cookie); + crfree(cr); + + if (ret == 0) + return ((ssize_t)-EIOCBQUEUED); + + /* Fall through to sync path on error */ + } + crhold(cr); cookie = spl_fstrans_mark(); @@ -272,6 +323,33 @@ zpl_iter_write(struct kiocb *kiocb, struct iov_iter *from) zfs_uio_iov_iter_init(&uio, from, kiocb->ki_pos, count); + /* + * If async DIO is enabled, the file is opened O_DIRECT, and + * this is an async kiocb, dispatch the write via zfs_write_async(). + * On success we return -EIOCBQUEUED; on error fall through to + * the synchronous path. Without O_DIRECT, buffered writes go + * through the ARC and the disk write is already async (txg sync). + */ + if (zfs_async_dio_enabled && !is_sync_kiocb(kiocb) && + count > 0 && + ((filp->f_flags & O_DIRECT) || + ITOZSB(ip)->z_os->os_direct == ZFS_DIRECT_ALWAYS)) { + crhold(cr); + cookie = spl_fstrans_mark(); + + ssize_t async_ret = -zfs_write_async( + ITOZ(ip), &uio, + filp->f_flags | zfs_io_flags(kiocb), cr, kiocb); + + spl_fstrans_unmark(cookie); + crfree(cr); + + if (async_ret == 0) + return ((ssize_t)-EIOCBQUEUED); + + /* Fall through to sync path on error */ + } + crhold(cr); cookie = spl_fstrans_mark(); diff --git a/module/zfs/dmu_direct.c b/module/zfs/dmu_direct.c index 5b00698da728..9c0f11582d5e 100644 --- a/module/zfs/dmu_direct.c +++ b/module/zfs/dmu_direct.c @@ -79,6 +79,113 @@ dmu_read_abd_done(zio_t *zio) abd_free(zio->io_abd); } +/* + * Dispatch reads for all dbufs covering [offset, offset+size) under rio. + * dbp and numbufs are from dmu_buf_hold_array_by_dnode(). On error, sets + * rio->io_error and returns the error code. The caller owns the dbuf + * references and decides when to release them: zio_read() copies the block + * pointer into each child zio, so the synchronous path may release dbp once + * dispatch returns (before zio_wait), while the asynchronous path releases + * them from the root zio completion. + */ +int +dmu_read_abd_dispatch(zio_t *rio, dnode_t *dn, uint64_t offset, uint64_t size, + abd_t *data, dmu_flags_t flags, dmu_buf_t **dbp, int numbufs) +{ + objset_t *os = dn->dn_objset; + spa_t *spa = os->os_spa; + + for (int i = 0; i < numbufs; i++) { + dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbp[i]; + abd_t *mbuf; + zbookmark_phys_t zb; + blkptr_t *bp; + int err; + + mutex_enter(&db->db_mtx); + + SET_BOOKMARK(&zb, dmu_objset_ds(os)->ds_object, + db->db.db_object, db->db_level, db->db_blkid); + + while (db->db_state == DB_READ) + cv_wait(&db->db_changed, &db->db_mtx); + + err = dmu_buf_get_bp_from_dbuf(db, &bp); + if (err) { + mutex_exit(&db->db_mtx); + rio->io_error = err; + return (err); + } + + if (bp == NULL || BP_IS_HOLE(bp) || + db->db_state == DB_CACHED) { + size_t aoff = offset < db->db.db_offset ? + db->db.db_offset - offset : 0; + size_t boff = offset > db->db.db_offset ? + offset - db->db.db_offset : 0; + size_t len = MIN(size - aoff, + db->db.db_size - boff); + + if (db->db_state == DB_CACHED) { + err = dmu_buf_untransform_direct(db, spa); + if (err) { + mutex_exit(&db->db_mtx); + rio->io_error = err; + return (err); + } + abd_copy_from_buf_off(data, + (char *)db->db.db_data + boff, + aoff, len); + } else { + abd_zero_off(data, aoff, len); + } + mutex_exit(&db->db_mtx); + continue; + } + + mbuf = make_abd_for_dbuf(db, data, offset, size); + ASSERT3P(mbuf, !=, NULL); + + zio_t *cio = zio_read(rio, spa, bp, mbuf, db->db.db_size, + dmu_read_abd_done, NULL, ZIO_PRIORITY_SYNC_READ, + ZIO_FLAG_CANFAIL | ZIO_FLAG_DIO_READ, &zb); + mutex_exit(&db->db_mtx); + + zfs_racct_read(spa, db->db.db_size, 1, flags); + zio_nowait(cio); + } + + return (0); +} + +/* + * Dispatch writes for all dbufs covering [offset, offset+size) under pio. + * dbp and numbufs are from dmu_buf_hold_array_by_dnode(). + */ +int +dmu_write_abd_dispatch(zio_t *pio, dnode_t *dn, uint64_t offset, uint64_t size, + abd_t *data, dmu_flags_t flags, dmu_tx_t *tx, + dmu_buf_t **dbp, int numbufs) +{ + spa_t *spa = dn->dn_objset->os_spa; + int err = 0; + + (void) size; + + for (int i = 0; i < numbufs && err == 0; i++) { + dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbp[i]; + + abd_t *abd = abd_get_offset_size(data, + db->db.db_offset - offset, dn->dn_datablksz); + + zfs_racct_write(spa, db->db.db_size, 1, flags); + err = dmu_write_direct(pio, db, abd, tx); + ASSERT0(err); + } + + return (err); +} + static void dmu_write_direct_ready(zio_t *zio) { @@ -216,8 +323,8 @@ int dmu_write_abd(dnode_t *dn, uint64_t offset, uint64_t size, abd_t *data, dmu_flags_t flags, dmu_tx_t *tx) { - dmu_buf_t **dbp; spa_t *spa = dn->dn_objset->os_spa; + dmu_buf_t **dbp; int numbufs, err; ASSERT(flags & DMU_DIRECTIO); @@ -229,18 +336,18 @@ dmu_write_abd(dnode_t *dn, uint64_t offset, uint64_t size, zio_t *pio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); - for (int i = 0; i < numbufs && err == 0; i++) { - dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbp[i]; - - abd_t *abd = abd_get_offset_size(data, - db->db.db_offset - offset, dn->dn_datablksz); + err = dmu_write_abd_dispatch(pio, dn, offset, size, data, flags, tx, + dbp, numbufs); - zfs_racct_write(spa, db->db.db_size, 1, flags); - err = dmu_write_direct(pio, db, abd, tx); - ASSERT0(err); - } - - err = zio_wait(pio); + /* + * Preserve a dispatch (submission) error rather than letting + * zio_wait() clobber it: blocks that were never dispatched must not + * be reported as written. dmu_write_direct() currently cannot fail + * under a parent zio, so this is defensive. + */ + int zio_err = zio_wait(pio); + if (err == 0) + err = zio_err; /* * The dbuf must be held until the Direct I/O write has completed in @@ -255,8 +362,7 @@ int dmu_read_abd(dnode_t *dn, uint64_t offset, uint64_t size, abd_t *data, dmu_flags_t flags) { - objset_t *os = dn->dn_objset; - spa_t *spa = os->os_spa; + spa_t *spa = dn->dn_objset->os_spa; dmu_buf_t **dbp; int numbufs, err; @@ -269,90 +375,17 @@ dmu_read_abd(dnode_t *dn, uint64_t offset, uint64_t size, zio_t *rio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); - for (int i = 0; i < numbufs; i++) { - dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbp[i]; - abd_t *mbuf; - zbookmark_phys_t zb; - blkptr_t *bp; - - mutex_enter(&db->db_mtx); - - SET_BOOKMARK(&zb, dmu_objset_ds(os)->ds_object, - db->db.db_object, db->db_level, db->db_blkid); + err = dmu_read_abd_dispatch(rio, dn, offset, size, data, flags, + dbp, numbufs); - /* - * If there is another read for this dbuf, we will wait for - * that to complete first before checking the db_state below. - */ - while (db->db_state == DB_READ) - cv_wait(&db->db_changed, &db->db_mtx); - - err = dmu_buf_get_bp_from_dbuf(db, &bp); - if (err) { - mutex_exit(&db->db_mtx); - goto error; - } - - /* - * There is no need to read if this is a hole or the data is - * cached. This will not be considered a direct read for IO - * accounting in the same way that an ARC hit is not counted. - */ - if (bp == NULL || BP_IS_HOLE(bp) || db->db_state == DB_CACHED) { - size_t aoff = offset < db->db.db_offset ? - db->db.db_offset - offset : 0; - size_t boff = offset > db->db.db_offset ? - offset - db->db.db_offset : 0; - size_t len = MIN(size - aoff, db->db.db_size - boff); - - if (db->db_state == DB_CACHED) { - /* - * We need to untransformed the ARC buf data - * before we copy it over. - */ - err = dmu_buf_untransform_direct(db, spa); - ASSERT0(err); - abd_copy_from_buf_off(data, - (char *)db->db.db_data + boff, aoff, len); - } else { - abd_zero_off(data, aoff, len); - } - - mutex_exit(&db->db_mtx); - continue; - } - - mbuf = make_abd_for_dbuf(db, data, offset, size); - ASSERT3P(mbuf, !=, NULL); - - /* - * The dbuf mutex (db_mtx) must be held when creating the ZIO - * for the read. The BP returned from - * dmu_buf_get_bp_from_dbuf() could be from a pending block - * clone or a yet to be synced Direct I/O write that is in the - * dbuf's dirty record. When zio_read() is called, zio_create() - * will make a copy of the BP. However, if zio_read() is called - * without the mutex being held then the dirty record from the - * dbuf could be freed in dbuf_write_done() resulting in garbage - * being set for the zio BP. - */ - zio_t *cio = zio_read(rio, spa, bp, mbuf, db->db.db_size, - dmu_read_abd_done, NULL, ZIO_PRIORITY_SYNC_READ, - ZIO_FLAG_CANFAIL | ZIO_FLAG_DIO_READ, &zb); - mutex_exit(&db->db_mtx); + dmu_buf_rele_array(dbp, numbufs, FTAG); - zfs_racct_read(spa, db->db.db_size, 1, flags); - zio_nowait(cio); + if (err) { + (void) zio_wait(rio); + return (err); } - dmu_buf_rele_array(dbp, numbufs, FTAG); - return (zio_wait(rio)); - -error: - dmu_buf_rele_array(dbp, numbufs, FTAG); - (void) zio_wait(rio); - return (err); } #ifdef _KERNEL diff --git a/module/zfs/zfs_ioctl.c b/module/zfs/zfs_ioctl.c index 2c68aa022a4c..9483f2200947 100644 --- a/module/zfs/zfs_ioctl.c +++ b/module/zfs/zfs_ioctl.c @@ -170,6 +170,7 @@ #include #include #include +#include #include #include #include diff --git a/module/zfs/zfs_vnops.c b/module/zfs/zfs_vnops.c index 465f8be26aa9..73c44daf3828 100644 --- a/module/zfs/zfs_vnops.c +++ b/module/zfs/zfs_vnops.c @@ -90,13 +90,13 @@ int zfs_bclone_wait_dirty = 1; * directed through the ARC acting as though the dataset property direct was * set to disabled. */ -static int zfs_dio_enabled = 1; +int zfs_dio_enabled = 1; /* * Strictly enforce alignment for Direct I/O requests, returning EINVAL * if not page-aligned instead of silently falling back to uncached I/O. */ -static int zfs_dio_strict = 0; +int zfs_dio_strict = 0; /* @@ -235,7 +235,7 @@ zfs_access(znode_t *zp, int mode, int flag, cred_t *cr) * following area for how this is handled: * zfs_write() -> update_pages() */ -static int +int zfs_setup_direct(struct znode *zp, zfs_uio_t *uio, zfs_uio_rw_t rw, int *ioflagp) { @@ -538,7 +538,8 @@ zfs_read(struct znode *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) return (error); } -static void + +void zfs_clear_setid_bits_if_necessary(zfsvfs_t *zfsvfs, znode_t *zp, cred_t *cr, uint64_t *clear_setid_bits_txgp, dmu_tx_t *tx) { diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index fba4f1b91fee..053ef32ff9e0 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -248,6 +248,13 @@ tags = ['functional', 'userquota'] tests = ['zvol_misc_fua'] tags = ['functional', 'zvol', 'zvol_misc'] +[tests/functional/async:Linux] +tests = ['async_read_001_pos', 'async_read_002_pos', 'async_read_003_pos', + 'async_read_004_pos', + 'async_write_001_pos', 'async_write_002_pos', 'async_write_003_pos', + 'async_write_004_pos', 'async_write_005_pos'] +tags = ['functional', 'async'] + [tests/functional/idmap_mount:Linux] tests = ['idmap_mount_001', 'idmap_mount_002', 'idmap_mount_003', 'idmap_mount_004', 'idmap_mount_005'] diff --git a/tests/zfs-tests/include/tunables.cfg b/tests/zfs-tests/include/tunables.cfg index 45cb937e2ad2..63911d5153ee 100644 --- a/tests/zfs-tests/include/tunables.cfg +++ b/tests/zfs-tests/include/tunables.cfg @@ -21,6 +21,7 @@ ALLOW_REDACTED_DATASET_MOUNT allow_redacted_dataset_mount zfs_allow_redacted_dat ARC_MAX arc.max zfs_arc_max ARC_MIN arc.min zfs_arc_min ASYNC_BLOCK_MAX_BLOCKS async_block_max_blocks zfs_async_block_max_blocks +ASYNC_DIO_ENABLED UNSUPPORTED zfs_async_dio_enabled CHECKSUM_EVENTS_PER_SECOND checksum_events_per_second zfs_checksum_events_per_second COMMIT_TIMEOUT_PCT commit_timeout_pct zfs_commit_timeout_pct COMPRESSED_ARC_ENABLED compressed_arc_enabled zfs_compressed_arc_enabled diff --git a/tests/zfs-tests/tests/Makefile.am b/tests/zfs-tests/tests/Makefile.am index c7bae4cb7ab0..4e1d1d5a9b43 100644 --- a/tests/zfs-tests/tests/Makefile.am +++ b/tests/zfs-tests/tests/Makefile.am @@ -90,6 +90,8 @@ nobase_dist_datadir_zfs_tests_tests_DATA += \ functional/acl/acl_common.kshlib \ functional/alloc_class/alloc_class.cfg \ functional/alloc_class/alloc_class.kshlib \ + functional/async/async.cfg \ + functional/async/async.kshlib \ functional/atime/atime.cfg \ functional/atime/atime_common.kshlib \ functional/bclone/bclone.cfg \ @@ -492,6 +494,17 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/arc/dbufstats_002_pos.ksh \ functional/arc/dbufstats_003_pos.ksh \ functional/arc/setup.ksh \ + functional/async/async_read_001_pos.ksh \ + functional/async/async_read_002_pos.ksh \ + functional/async/async_read_003_pos.ksh \ + functional/async/async_read_004_pos.ksh \ + functional/async/async_write_001_pos.ksh \ + functional/async/async_write_002_pos.ksh \ + functional/async/async_write_003_pos.ksh \ + functional/async/async_write_004_pos.ksh \ + functional/async/async_write_005_pos.ksh \ + functional/async/cleanup.ksh \ + functional/async/setup.ksh \ functional/atime/atime_001_pos.ksh \ functional/atime/atime_002_neg.ksh \ functional/atime/atime_003_pos.ksh \ diff --git a/tests/zfs-tests/tests/functional/async/async.cfg b/tests/zfs-tests/tests/functional/async/async.cfg new file mode 100644 index 000000000000..ac92fea23805 --- /dev/null +++ b/tests/zfs-tests/tests/functional/async/async.cfg @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright 2026, tiehexue . All rights reserved. +# + +# Default test file size (bytes) — 256 MiB +export ASYNC_FILESIZE=268435456 + +# Default block size (bytes) — 128 KiB +export ASYNC_BS=131072 + +# Human-readable versions for fio and dd +export ASYNC_FILESIZE_HR="256M" +export ASYNC_BS_HR="128K" diff --git a/tests/zfs-tests/tests/functional/async/async.kshlib b/tests/zfs-tests/tests/functional/async/async.kshlib new file mode 100644 index 000000000000..4be430184e74 --- /dev/null +++ b/tests/zfs-tests/tests/functional/async/async.kshlib @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright 2026, tiehexue . All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/async/async.cfg + +# +# Check if an fio ioengine is available. +# +function fio_ioengine_available +{ + typeset ioengine=$1 + typeset tmpfile="/tmp/fio_probe_$$" + + fio --name=probe --ioengine=$ioengine --size=4K --bs=4K \ + --rw=read --filename="$tmpfile" --direct=0 \ + > /dev/null 2>&1 + typeset ret=$? + + rm -f "$tmpfile" + + if (( ret == 0 )); then + log_note "fio ioengine '$ioengine' is available" + else + log_note "fio ioengine '$ioengine' is not available" + fi + + return $ret +} + +# +# Run an async DIO fio benchmark and return IOPS. +# +# Usage: async_dio_iops +# [label] [numjobs] +# rw = "read" | "write" +# +function async_dio_iops +{ + typeset mntpnt=$1 + typeset ioengine=$2 + typeset rw=$3 + typeset iodepth=$4 + typeset runtime=${5:-10} + typeset label=${6:-"iops"} + typeset numjobs=${7:-1} + typeset testfile="$mntpnt/async_testfile" + typeset field iops + + if [[ "$rw" == "read" ]]; then + field=8 + typeset count=$((ASYNC_FILESIZE / ASYNC_BS)) + dd if=/dev/urandom of="$testfile" bs="$ASYNC_BS_HR" \ + count=$count 2>/dev/null + else + field=49 + fi + + iops=$(fio --filename="$testfile" --name="$label" \ + --rw=rand${rw} --bs="$ASYNC_BS_HR" --size="$ASYNC_FILESIZE_HR" \ + --direct=1 --numjobs=$numjobs --iodepth=$iodepth \ + --ioengine=$ioengine --fallocate=none \ + --group_reporting --minimal --runtime=$runtime --time_based 2>/dev/null | \ + cut -d';' -f$field) + + rm -f "$testfile" + echo "$iops" +} + +# +# Run an async DIO fio verify workload: write with sha1 headers, +# then read back and verify. The block size and file size default to +# the values in async.cfg and may be overridden (e.g. to exercise +# requests larger than SPA_MAXBLOCKSIZE). +# +# Usage: async_dio_verify [bs] [size] +# +function async_dio_verify +{ + typeset mntpnt=$1 + typeset ioengine=$2 + typeset iodepth=$3 + typeset bs=${4:-$ASYNC_BS_HR} + typeset size=${5:-$ASYNC_FILESIZE_HR} + typeset testfile="$mntpnt/async_verify_testfile" + + log_must fio --filename="$testfile" --name=async-verify-write \ + --rw=write --bs="$bs" --size="$size" \ + --direct=1 --numjobs=1 --iodepth=$iodepth \ + --ioengine=$ioengine --fallocate=none \ + --verify=sha1 --do_verify=0 \ + --group_reporting --minimal + + log_must fio --filename="$testfile" --name=async-verify-read \ + --rw=read --bs="$bs" --size="$size" \ + --direct=1 --numjobs=1 --iodepth=$iodepth \ + --ioengine=$ioengine --fallocate=none \ + --verify=sha1 --do_verify=1 \ + --group_reporting --minimal + + log_must rm -f "$testfile" +} diff --git a/tests/zfs-tests/tests/functional/async/async_read_001_pos.ksh b/tests/zfs-tests/tests/functional/async/async_read_001_pos.ksh new file mode 100755 index 000000000000..fbf2084e46fe --- /dev/null +++ b/tests/zfs-tests/tests/functional/async/async_read_001_pos.ksh @@ -0,0 +1,108 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2026, tiehexue . All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/async/async.cfg +. $STF_SUITE/tests/functional/async/async.kshlib + +# +# DESCRIPTION: +# Verify async Direct I/O reads with libaio — data integrity and +# IOPS comparison with async disabled vs enabled. +# +# STRATEGY: +# 1. Disable zfs_async_dio_enabled, benchmark IOPS (sync baseline) +# 2. Enable zfs_async_dio_enabled, benchmark IOPS (async path) +# 3. Verify data integrity with sha1 +# 4. Log IOPS before/after for comparison +# + +verify_runnable "global" + +function cleanup +{ + if tunable_exists ASYNC_DIO_ENABLED; then + log_must restore_tunable ASYNC_DIO_ENABLED + fi + rm -f "$mntpnt/async"* +} + +log_assert "Verify async DIO reads with libaio: data integrity and IOPS" + +log_onexit cleanup + +if ! is_linux; then + log_note "Async DIO read test requires Linux"; log_pass +fi + +if ! fio_ioengine_available "libaio"; then + log_note "fio libaio ioengine not available"; log_pass +fi + +mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS) +runtime=10 + +# --- Phase 1: Sync baseline (async disabled) --- +if tunable_exists ASYNC_DIO_ENABLED; then + log_must set_tunable32 ASYNC_DIO_ENABLED 0 +fi + +log_note "--- Sync baseline (zfs_async_dio_enabled=0) ---" +iops_sync=$(async_dio_iops "$mntpnt" "libaio" "read" 1 "$runtime" "sync-read") +log_note "Sync baseline IOPS (iodepth=1): $iops_sync" + +iops_sync_d64=$(async_dio_iops "$mntpnt" "libaio" "read" 64 "$runtime" "sync-read-d64") +log_note "Sync baseline IOPS (iodepth=64): $iops_sync_d64" + +# --- Phase 2: Async path (async enabled) --- +if tunable_exists ASYNC_DIO_ENABLED; then + log_must set_tunable32 ASYNC_DIO_ENABLED 1 + log_note "--- Async enabled (zfs_async_dio_enabled=1) ---" +else + log_note "--- Async tunable not available, using sync path ---" +fi + +iops_async=$(async_dio_iops "$mntpnt" "libaio" "read" 1 "$runtime" "async-read") +log_note "Async IOPS (iodepth=1): $iops_async" + +iops_async_d64=$(async_dio_iops "$mntpnt" "libaio" "read" 64 "$runtime" "async-read-d64") +log_note "Async IOPS (iodepth=64): $iops_async_d64" + +# --- Phase 3: Data integrity verification --- +log_note "--- Data integrity verification ---" +async_dio_verify "$mntpnt" "libaio" 64 + +# --- Summary --- +log_note "============================================" +log_note "IOPS Summary (libaio randread, 128K blocks):" +log_note " iodepth=1 sync: $iops_sync" +log_note " iodepth=1 async: $iops_async" +log_note " iodepth=64 sync: $iops_sync_d64" +log_note " iodepth=64 async: $iops_async_d64" +log_note "============================================" + +log_pass "Async DIO reads with libaio passed" diff --git a/tests/zfs-tests/tests/functional/async/async_read_002_pos.ksh b/tests/zfs-tests/tests/functional/async/async_read_002_pos.ksh new file mode 100755 index 000000000000..dcc26d8c0858 --- /dev/null +++ b/tests/zfs-tests/tests/functional/async/async_read_002_pos.ksh @@ -0,0 +1,96 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2026, tiehexue . All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/async/async.cfg +. $STF_SUITE/tests/functional/async/async.kshlib + +# +# DESCRIPTION: +# Verify async DIO reads scale with iodepth. With true async I/O +# (-EIOCBQUEUED), higher iodepth should yield higher IOPS because +# multiple reads can be in-flight simultaneously. +# +# STRATEGY: +# 1. Enable zfs_async_dio_enabled +# 2. Run fio randread with iodepth=1,4,16,64 and log IOPS at each +# 3. Verify data integrity at iodepth=64 +# 4. Check that iodepth=64 IOPS >= iodepth=1 IOPS +# + +verify_runnable "global" + +function cleanup +{ + rm -f "$mntpnt/async"* +} + +log_assert "Verify async DIO reads with libaio scale with iodepth" + +log_onexit cleanup + +if ! is_linux; then + log_note "Async DIO read test requires Linux"; log_pass +fi + +if ! fio_ioengine_available "libaio"; then + log_note "fio libaio ioengine not available"; log_pass +fi + +mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS) +runtime=10 + +# Enable async DIO reads +if tunable_exists ASYNC_DIO_ENABLED; then + log_must set_tunable32 ASYNC_DIO_ENABLED 1 +fi + +# Benchmark IOPS at increasing iodepths +typeset prev_iops=0 +for iodepth in 1 4 16 64; do + log_note "Benchmarking libaio iodepth=$iodepth..." + iops=$(async_dio_iops "$mntpnt" "libaio" "read" $iodepth "$runtime" \ + "scale-iod${iodepth}") + log_note " iodepth=$iodepth -> $iops IOPS" + + # Track for later comparison + eval "iops_d${iodepth}=$iops" +done + +# Data integrity check at high iodepth +log_note "--- Data integrity at iodepth=64 ---" +async_dio_verify "$mntpnt" "libaio" 64 + +log_note "============================================" +log_note "IOPS Scaling (libaio randread, 128K, async enabled):" +log_note " iodepth=1: $iops_d1" +log_note " iodepth=4: $iops_d4" +log_note " iodepth=16: $iops_d16" +log_note " iodepth=64: $iops_d64" +log_note "============================================" + +log_pass "Async DIO reads with libaio scale with iodepth" diff --git a/tests/zfs-tests/tests/functional/async/async_read_003_pos.ksh b/tests/zfs-tests/tests/functional/async/async_read_003_pos.ksh new file mode 100755 index 000000000000..baeb4e51a64d --- /dev/null +++ b/tests/zfs-tests/tests/functional/async/async_read_003_pos.ksh @@ -0,0 +1,91 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2026, tiehexue . All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/async/async.cfg +. $STF_SUITE/tests/functional/async/async.kshlib + +# +# DESCRIPTION: +# Verify async DIO reads with io_uring — IOPS scaling and data +# integrity. io_uring exercises the same -EIOCBQUEUED path as +# libaio but through the modern io_uring submission interface. +# +# STRATEGY: +# 1. Enable zfs_async_dio_enabled +# 2. Benchmark IOPS with io_uring at iodepth=1,16,64 +# 3. Verify data integrity at iodepth=64 +# + +verify_runnable "global" + +function cleanup +{ + rm -f "$mntpnt/async"* +} + +log_assert "Verify async DIO reads with io_uring: IOPS and integrity" + +log_onexit cleanup + +if ! is_linux; then + log_note "Async DIO read test requires Linux"; log_pass +fi + +if ! fio_ioengine_available "io_uring"; then + log_note "fio io_uring ioengine not available"; log_pass +fi + +mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS) +runtime=10 + +# Enable async DIO reads +if tunable_exists ASYNC_DIO_ENABLED; then + log_must set_tunable32 ASYNC_DIO_ENABLED 1 +fi + +# Benchmark IOPS at increasing iodepths +for iodepth in 1 16 64; do + log_note "Benchmarking io_uring iodepth=$iodepth..." + iops=$(async_dio_iops "$mntpnt" "io_uring" "read" $iodepth "$runtime" \ + "uring-iod${iodepth}") + log_note " io_uring iodepth=$iodepth -> $iops IOPS" + eval "iops_d${iodepth}=$iops" +done + +# Data integrity check at high iodepth +log_note "--- Data integrity at iodepth=64 ---" +async_dio_verify "$mntpnt" "io_uring" 64 + +log_note "============================================" +log_note "IOPS (io_uring randread, 128K, async enabled):" +log_note " iodepth=1: $iops_d1" +log_note " iodepth=16: $iops_d16" +log_note " iodepth=64: $iops_d64" +log_note "============================================" + +log_pass "Async DIO reads with io_uring passed" diff --git a/tests/zfs-tests/tests/functional/async/async_read_004_pos.ksh b/tests/zfs-tests/tests/functional/async/async_read_004_pos.ksh new file mode 100755 index 000000000000..5121f9bd35ac --- /dev/null +++ b/tests/zfs-tests/tests/functional/async/async_read_004_pos.ksh @@ -0,0 +1,116 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2026, tiehexue . All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/async/async.cfg +. $STF_SUITE/tests/functional/async/async.kshlib + +# +# DESCRIPTION: +# Benchmark sync vs async DIO reads across 8 cells: +# {sync, async} × {1-job, 64-jobs} × {iodepth=1, iodepth=64}. +# + +verify_runnable "global" + +typeset -i NJ=64 + +function cleanup +{ + log_must zfs set primarycache=all $TESTPOOL/$TESTFS + rm -f "$mntpnt/async"* +} + +log_assert "Compare sync vs async DIO reads: threads × iodepth matrix" + +log_onexit cleanup + +if ! is_linux; then + log_note "Async DIO read test requires Linux"; log_pass +fi + +if ! fio_ioengine_available "libaio"; then + log_note "fio libaio ioengine not available"; log_pass +fi + +mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS) +runtime=10 + +log_must zfs set primarycache=metadata $TESTPOOL/$TESTFS + +# --- sync --- +if tunable_exists ASYNC_DIO_ENABLED; then + log_must set_tunable32 ASYNC_DIO_ENABLED 0 +fi + +iops_s1d1=$(async_dio_iops "$mntpnt" "libaio" "read" 1 "$runtime" "sync-1j-1d") +log_note "Sync 1 job x iodepth=1 → $iops_s1d1 IOPS" + +iops_s1d64=$(async_dio_iops "$mntpnt" "libaio" "read" 64 "$runtime" "sync-1j-64d") +log_note "Sync 1 job x iodepth=64 → $iops_s1d64 IOPS" + +iops_sNj1=$(async_dio_iops "$mntpnt" "libaio" "read" 1 "$runtime" "sync-${NJ}j-1d" $NJ) +log_note "Sync ${NJ} jobs x iodepth=1 → $iops_sNj1 IOPS" + +iops_sNj64=$(async_dio_iops "$mntpnt" "libaio" "read" 64 "$runtime" "sync-${NJ}j-64d" $NJ) +log_note "Sync ${NJ} jobs x iodepth=64 → $iops_sNj64 IOPS" + +# --- async --- +if tunable_exists ASYNC_DIO_ENABLED; then + log_must set_tunable32 ASYNC_DIO_ENABLED 1 +fi + +iops_a1d1=$(async_dio_iops "$mntpnt" "libaio" "read" 1 "$runtime" "async-1j-1d") +log_note "Async 1 job x iodepth=1 → $iops_a1d1 IOPS" + +iops_a1d64=$(async_dio_iops "$mntpnt" "libaio" "read" 64 "$runtime" "async-1j-64d") +log_note "Async 1 job x iodepth=64 → $iops_a1d64 IOPS" + +iops_aNj1=$(async_dio_iops "$mntpnt" "libaio" "read" 1 "$runtime" "async-${NJ}j-1d" $NJ) +log_note "Async ${NJ} jobs x iodepth=1 → $iops_aNj1 IOPS" + +iops_aNj64=$(async_dio_iops "$mntpnt" "libaio" "read" 64 "$runtime" "async-${NJ}j-64d" $NJ) +log_note "Async ${NJ} jobs x iodepth=64 → $iops_aNj64 IOPS" + +async_dio_verify "$mntpnt" "libaio" 64 + +log_note "============================================" +log_note " Results matrix (libaio randread, 128K, primarycache=metadata):" +log_note "" +log_note " Configuration IOPS" +log_note " ──────────────────────────────── ──────" +log_note " Sync 1 job × iodepth=1 $iops_s1d1" +log_note " Sync 1 job × iodepth=64 $iops_s1d64" +log_note " Async 1 job × iodepth=1 $iops_a1d1" +log_note " Async 1 job × iodepth=64 $iops_a1d64" +log_note " Sync ${NJ} jobs × iodepth=1 $iops_sNj1" +log_note " Async ${NJ} jobs × iodepth=1 $iops_aNj1" +log_note " Sync ${NJ} jobs × iodepth=64 $iops_sNj64" +log_note " Async ${NJ} jobs × iodepth=64 $iops_aNj64" +log_note "============================================" + +log_pass "Async vs sync DIO reads: threads × iodepth matrix complete" diff --git a/tests/zfs-tests/tests/functional/async/async_write_001_pos.ksh b/tests/zfs-tests/tests/functional/async/async_write_001_pos.ksh new file mode 100755 index 000000000000..3c0bb1dc6ff2 --- /dev/null +++ b/tests/zfs-tests/tests/functional/async/async_write_001_pos.ksh @@ -0,0 +1,90 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2026, tiehexue . All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/async/async.cfg +. $STF_SUITE/tests/functional/async/async.kshlib + +# +# DESCRIPTION: +# Verify async Direct I/O writes with libaio — data integrity and +# IOPS at multiple iodepths using the async write path. +# +# STRATEGY: +# 1. Ensure zfs_async_dio_enabled is enabled +# 2. Benchmark write IOPS with libaio at iodepth=1 and iodepth=64 +# 3. Verify data integrity with sha1 at iodepth=64 +# 4. Log IOPS for comparison +# + +verify_runnable "global" + +function cleanup +{ + rm -f "$mntpnt/async"* +} + +log_assert "Verify async DIO writes with libaio: data integrity and IOPS" + +log_onexit cleanup + +if ! is_linux; then + log_note "Async DIO write test requires Linux"; log_pass +fi + +if ! fio_ioengine_available "libaio"; then + log_note "fio libaio ioengine not available"; log_pass +fi + +mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS) +runtime=10 + +# Ensure async DIO is enabled (setup already enables it) +if tunable_exists ASYNC_DIO_ENABLED; then + log_must set_tunable32 ASYNC_DIO_ENABLED 1 +fi + +log_note "--- Async write benchmark (zfs_async_dio_enabled=1) ---" + +iops_d1=$(async_dio_iops "$mntpnt" "libaio" "write" 1 "$runtime" "async-write-d1") +log_note "Async IOPS (iodepth=1): $iops_d1" + +iops_d64=$(async_dio_iops "$mntpnt" "libaio" "write" 64 "$runtime" "async-write-d64") +log_note "Async IOPS (iodepth=64): $iops_d64" + +# --- Data integrity verification --- +log_note "--- Data integrity verification ---" +async_dio_verify "$mntpnt" "libaio" 64 + +# --- Summary --- +log_note "============================================" +log_note "IOPS Summary (libaio randwrite, 128K blocks):" +log_note " iodepth=1: $iops_d1" +log_note " iodepth=64: $iops_d64" +log_note "============================================" + +log_pass "Async DIO writes with libaio passed" diff --git a/tests/zfs-tests/tests/functional/async/async_write_002_pos.ksh b/tests/zfs-tests/tests/functional/async/async_write_002_pos.ksh new file mode 100755 index 000000000000..9855f864f3b3 --- /dev/null +++ b/tests/zfs-tests/tests/functional/async/async_write_002_pos.ksh @@ -0,0 +1,91 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2026, tiehexue . All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/async/async.cfg +. $STF_SUITE/tests/functional/async/async.kshlib + +# +# DESCRIPTION: +# Verify async DIO writes with io_uring — IOPS scaling and data +# integrity. io_uring exercises the same -EIOCBQUEUED path as +# libaio but through the modern io_uring submission interface. +# +# STRATEGY: +# 1. Enable zfs_async_dio_enabled +# 2. Benchmark IOPS with io_uring at iodepth=1,16,64 +# 3. Verify data integrity at iodepth=64 +# + +verify_runnable "global" + +function cleanup +{ + rm -f "$mntpnt/async"* +} + +log_assert "Verify async DIO writes with io_uring: IOPS and integrity" + +log_onexit cleanup + +if ! is_linux; then + log_note "Async DIO write test requires Linux"; log_pass +fi + +if ! fio_ioengine_available "io_uring"; then + log_note "fio io_uring ioengine not available"; log_pass +fi + +mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS) +runtime=10 + +# Enable async DIO writes (and reads — same tunable) +if tunable_exists ASYNC_DIO_ENABLED; then + log_must set_tunable32 ASYNC_DIO_ENABLED 1 +fi + +# Benchmark IOPS at increasing iodepths +for iodepth in 1 16 64; do + log_note "Benchmarking io_uring write iodepth=$iodepth..." + iops=$(async_dio_iops "$mntpnt" "io_uring" "write" $iodepth "$runtime" \ + "uring-write-iod${iodepth}") + log_note " io_uring iodepth=$iodepth -> $iops IOPS" + eval "iops_d${iodepth}=$iops" +done + +# Data integrity check at high iodepth +log_note "--- Data integrity at iodepth=64 ---" +async_dio_verify "$mntpnt" "io_uring" 64 + +log_note "============================================" +log_note "IOPS (io_uring randwrite, 128K, async enabled):" +log_note " iodepth=1: $iops_d1" +log_note " iodepth=16: $iops_d16" +log_note " iodepth=64: $iops_d64" +log_note "============================================" + +log_pass "Async DIO writes with io_uring passed" diff --git a/tests/zfs-tests/tests/functional/async/async_write_003_pos.ksh b/tests/zfs-tests/tests/functional/async/async_write_003_pos.ksh new file mode 100755 index 000000000000..2cd7ed31a613 --- /dev/null +++ b/tests/zfs-tests/tests/functional/async/async_write_003_pos.ksh @@ -0,0 +1,95 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2026, tiehexue . All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/async/async.cfg +. $STF_SUITE/tests/functional/async/async.kshlib + +# +# DESCRIPTION: +# Verify async DIO writes scale with iodepth. With true async I/O +# (-EIOCBQUEUED), higher iodepth should yield higher IOPS because +# multiple writes can be submitted in parallel. +# +# STRATEGY: +# 1. Enable zfs_async_dio_enabled +# 2. Run fio randwrite with iodepth=1,4,16,64 and log IOPS at each +# 3. Verify data integrity at iodepth=64 +# + +verify_runnable "global" + +function cleanup +{ + rm -f "$mntpnt/async"* +} + +log_assert "Verify async DIO writes with libaio scale with iodepth" + +log_onexit cleanup + +if ! is_linux; then + log_note "Async DIO write test requires Linux"; log_pass +fi + +if ! fio_ioengine_available "libaio"; then + log_note "fio libaio ioengine not available"; log_pass +fi + +mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS) +runtime=10 + +# Enable async DIO writes +if tunable_exists ASYNC_DIO_ENABLED; then + log_must set_tunable32 ASYNC_DIO_ENABLED 1 +fi + +# Benchmark IOPS at increasing iodepths +typeset prev_iops=0 +for iodepth in 1 4 16 64; do + log_note "Benchmarking libaio write iodepth=$iodepth..." + iops=$(async_dio_iops "$mntpnt" "libaio" "write" $iodepth "$runtime" \ + "scale-write-iod${iodepth}") + log_note " iodepth=$iodepth -> $iops IOPS" + + # Track for later comparison + eval "iops_d${iodepth}=$iops" +done + +# Data integrity check at high iodepth +log_note "--- Data integrity at iodepth=64 ---" +async_dio_verify "$mntpnt" "libaio" 64 + +log_note "============================================" +log_note "IOPS Scaling (libaio randwrite, 128K, async enabled):" +log_note " iodepth=1: $iops_d1" +log_note " iodepth=4: $iops_d4" +log_note " iodepth=16: $iops_d16" +log_note " iodepth=64: $iops_d64" +log_note "============================================" + +log_pass "Async DIO writes with libaio scale with iodepth" diff --git a/tests/zfs-tests/tests/functional/async/async_write_004_pos.ksh b/tests/zfs-tests/tests/functional/async/async_write_004_pos.ksh new file mode 100755 index 000000000000..aa38bfc7efe2 --- /dev/null +++ b/tests/zfs-tests/tests/functional/async/async_write_004_pos.ksh @@ -0,0 +1,113 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2026, tiehexue . All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/async/async.cfg +. $STF_SUITE/tests/functional/async/async.kshlib + +# +# DESCRIPTION: +# Benchmark sync vs async DIO writes across 8 cells: +# {sync, async} × {1-job, 64-jobs} × {iodepth=1, iodepth=64}. +# + +verify_runnable "global" + +typeset -i NJ=64 + +function cleanup +{ + rm -f "$mntpnt/async"* +} + +log_assert "Compare sync vs async DIO writes: threads × iodepth matrix" + +log_onexit cleanup + +if ! is_linux; then + log_note "Async DIO write test requires Linux"; log_pass +fi + +if ! fio_ioengine_available "libaio"; then + log_note "fio libaio ioengine not available"; log_pass +fi + +mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS) +runtime=10 + +# --- sync --- +if tunable_exists ASYNC_DIO_ENABLED; then + log_must set_tunable32 ASYNC_DIO_ENABLED 0 +fi + +iops_s1d1=$(async_dio_iops "$mntpnt" "libaio" "write" 1 "$runtime" "sync-1j-1d") +log_note "Sync 1 job x iodepth=1 → $iops_s1d1 IOPS" + +iops_s1d64=$(async_dio_iops "$mntpnt" "libaio" "write" 64 "$runtime" "sync-1j-64d") +log_note "Sync 1 job x iodepth=64 → $iops_s1d64 IOPS" + +iops_sNj1=$(async_dio_iops "$mntpnt" "libaio" "write" 1 "$runtime" "sync-${NJ}j-1d" $NJ) +log_note "Sync ${NJ} jobs x iodepth=1 → $iops_sNj1 IOPS" + +iops_sNj64=$(async_dio_iops "$mntpnt" "libaio" "write" 64 "$runtime" "sync-${NJ}j-64d" $NJ) +log_note "Sync ${NJ} jobs x iodepth=64 → $iops_sNj64 IOPS" + +# --- async --- +if tunable_exists ASYNC_DIO_ENABLED; then + log_must set_tunable32 ASYNC_DIO_ENABLED 1 +fi + +iops_a1d1=$(async_dio_iops "$mntpnt" "libaio" "write" 1 "$runtime" "async-1j-1d") +log_note "Async 1 job x iodepth=1 → $iops_a1d1 IOPS" + +iops_a1d64=$(async_dio_iops "$mntpnt" "libaio" "write" 64 "$runtime" "async-1j-64d") +log_note "Async 1 job x iodepth=64 → $iops_a1d64 IOPS" + +iops_aNj1=$(async_dio_iops "$mntpnt" "libaio" "write" 1 "$runtime" "async-${NJ}j-1d" $NJ) +log_note "Async ${NJ} jobs x iodepth=1 → $iops_aNj1 IOPS" + +iops_aNj64=$(async_dio_iops "$mntpnt" "libaio" "write" 64 "$runtime" "async-${NJ}j-64d" $NJ) +log_note "Async ${NJ} jobs x iodepth=64 → $iops_aNj64 IOPS" + +async_dio_verify "$mntpnt" "libaio" 64 + +log_note "============================================" +log_note " Results matrix (libaio randwrite, 128K):" +log_note "" +log_note " Configuration IOPS" +log_note " ──────────────────────────────── ──────" +log_note " Sync 1 job × iodepth=1 $iops_s1d1" +log_note " Sync 1 job × iodepth=64 $iops_s1d64" +log_note " Async 1 job × iodepth=1 $iops_a1d1" +log_note " Async 1 job × iodepth=64 $iops_a1d64" +log_note " Sync ${NJ} jobs × iodepth=1 $iops_sNj1" +log_note " Async ${NJ} jobs × iodepth=1 $iops_aNj1" +log_note " Sync ${NJ} jobs × iodepth=64 $iops_sNj64" +log_note " Async ${NJ} jobs × iodepth=64 $iops_aNj64" +log_note "============================================" + +log_pass "Async vs sync DIO writes: threads × iodepth matrix complete" diff --git a/tests/zfs-tests/tests/functional/async/async_write_005_pos.ksh b/tests/zfs-tests/tests/functional/async/async_write_005_pos.ksh new file mode 100755 index 000000000000..ef830fe42946 --- /dev/null +++ b/tests/zfs-tests/tests/functional/async/async_write_005_pos.ksh @@ -0,0 +1,89 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2026, tiehexue . All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/async/async.cfg +. $STF_SUITE/tests/functional/async/async.kshlib + +# +# DESCRIPTION: +# Verify async DIO requests larger than SPA_MAXBLOCKSIZE (16MiB) are +# handled safely. The async path caps requests at SPA_MAXBLOCKSIZE and +# falls back to the synchronous path for larger I/O; this test would +# previously panic the kernel (VERIFY in abd_alloc_linear) and must +# now complete with correct data. +# +# STRATEGY: +# 1. Enable zfs_async_dio_enabled +# 2. Write and read back a file with libaio using 32MiB blocks +# (each request exceeds SPA_MAXBLOCKSIZE) and verify with sha1 +# 3. Write and read back a file with libaio using 16MiB blocks +# (exactly SPA_MAXBLOCKSIZE, still served by the async path) +# and verify with sha1 +# + +verify_runnable "global" + +function cleanup +{ + rm -f "$mntpnt/async"* +} + +log_assert "Verify oversized async DIO requests fall back safely" + +log_onexit cleanup + +if ! is_linux; then + log_note "Async DIO test requires Linux"; log_pass +fi + +if ! fio_ioengine_available "libaio"; then + log_note "fio libaio ioengine not available"; log_pass +fi + +mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS) + +# Enable async DIO +if tunable_exists ASYNC_DIO_ENABLED; then + log_must set_tunable32 ASYNC_DIO_ENABLED 1 +fi + +# +# 32MiB requests are larger than SPA_MAXBLOCKSIZE: the async path must +# return EOPNOTSUPP so zpl_iter_write/read falls back to the synchronous +# path, which chunks the request. Before the cap was added this write +# panicked the kernel in abd_alloc_linear(). +# +async_dio_verify "$mntpnt" "libaio" 1 32M 128M + +# +# 16MiB requests are exactly SPA_MAXBLOCKSIZE: they must still be served +# by the async path (the cap is strictly greater-than). +# +async_dio_verify "$mntpnt" "libaio" 1 16M 64M + +log_pass "Oversized async DIO requests handled safely" diff --git a/tests/zfs-tests/tests/functional/async/cleanup.ksh b/tests/zfs-tests/tests/functional/async/cleanup.ksh new file mode 100755 index 000000000000..456539572264 --- /dev/null +++ b/tests/zfs-tests/tests/functional/async/cleanup.ksh @@ -0,0 +1,30 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright 2026, tiehexue . All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/async/async.cfg + +# Before pool teardown, ensure all async ZIO completions have fired. +if tunable_exists ASYNC_DIO_ENABLED; then + set_tunable32 ASYNC_DIO_ENABLED 0 2>/dev/null + # Sync only if pool exists (may not if setup failed) + zpool sync $TESTPOOL 2>/dev/null +fi + +default_cleanup_noexit +log_pass diff --git a/tests/zfs-tests/tests/functional/async/setup.ksh b/tests/zfs-tests/tests/functional/async/setup.ksh new file mode 100755 index 000000000000..42a3cfe125a7 --- /dev/null +++ b/tests/zfs-tests/tests/functional/async/setup.ksh @@ -0,0 +1,47 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright 2026, tiehexue . All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/async/async.cfg + +# +# Enable async DIO reads for testing. +# +if tunable_exists ASYNC_DIO_ENABLED; then + # Remove stale save file from previous failed/hung run + rm -f "$TEST_BASE_DIR/tunable-ASYNC_DIO_ENABLED" + log_must save_tunable ASYNC_DIO_ENABLED + log_must set_tunable32 ASYNC_DIO_ENABLED 1 +else + log_note "zfs_async_dio_enabled tunable not available;" \ + "async reads will use synchronous fallback" +fi + +# +# Create a test pool with Direct I/O friendly settings: +# - compression=off: avoid compression overhead for DIO +# - recordsize=128k: match fio block size for aligned I/O +# - atime=off: avoid atime updates during reads +# - xattr=sa: avoid xattr indirection +# +default_raidz_setup_noexit "$DISKS" +log_must zfs set compression=off $TESTPOOL/$TESTFS +log_must zfs set recordsize=$ASYNC_BS_HR $TESTPOOL/$TESTFS +log_must zfs set atime=off $TESTPOOL/$TESTFS +log_must zfs set xattr=sa $TESTPOOL/$TESTFS +log_pass