true async io in linux

config/kernel-ki_complete.m4

@@ -0,0 +1,26 @@
+dnl # SPDX-License-Identifier: CDDL-1.0
+dnl #
+dnl # 5.19 API change,
+dnl # kiocb->ki_complete() reduced from 3 args to 2:
+dnl #   old: void (*ki_complete)(struct kiocb *, long, long)
+dnl #   new: void (*ki_complete)(struct kiocb *, long)
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_KIOCB_KI_COMPLETE], [
+	ZFS_LINUX_TEST_SRC([kiocb_ki_complete_2args], [
+		#include <linux/fs.h>
+	],[
+		struct kiocb *kiocb = NULL;
+		kiocb->ki_complete(kiocb, 0);
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_KIOCB_KI_COMPLETE], [
+	AC_MSG_CHECKING([whether kiocb->ki_complete() wants 2 args])
+	ZFS_LINUX_TEST_RESULT([kiocb_ki_complete_2args], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_2ARGS_KI_COMPLETE, 1,
+		    [kiocb->ki_complete() wants 2 args])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])

config/kernel.m4

@@ -90,6 +90,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [
 	ZFS_AC_KERNEL_SRC_VFS_REMAP_FILE_RANGE
 	ZFS_AC_KERNEL_SRC_VFS_CLONE_FILE_RANGE
 	ZFS_AC_KERNEL_SRC_VFS_DEDUPE_FILE_RANGE
+	ZFS_AC_KERNEL_SRC_KIOCB_KI_COMPLETE
 	ZFS_AC_KERNEL_SRC_KMAP_ATOMIC_ARGS
 	ZFS_AC_KERNEL_SRC_KMAP_LOCAL_PAGE
 	ZFS_AC_KERNEL_SRC_FOLLOW_DOWN_ONE
@@ -204,6 +205,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [
 	ZFS_AC_KERNEL_VFS_REMAP_FILE_RANGE
 	ZFS_AC_KERNEL_VFS_CLONE_FILE_RANGE
 	ZFS_AC_KERNEL_VFS_DEDUPE_FILE_RANGE
+	ZFS_AC_KERNEL_KIOCB_KI_COMPLETE
 	ZFS_AC_KERNEL_KMAP_ATOMIC_ARGS
 	ZFS_AC_KERNEL_KMAP_LOCAL_PAGE
 	ZFS_AC_KERNEL_FOLLOW_DOWN_ONE

include/os/linux/Makefile.am

@@ -24,6 +24,7 @@ kernel_sysdir = $(kerneldir)/sys
 kernel_sys_HEADERS = \
 	%D%/zfs/sys/abd_os.h \
 	%D%/zfs/sys/abd_impl_os.h \
+	%D%/zfs/sys/dmu_direct_os.h \
 	%D%/zfs/sys/policy.h \
 	%D%/zfs/sys/trace_acl.h \
 	%D%/zfs/sys/trace_arc.h \

include/os/linux/spl/sys/uio.h

@@ -189,6 +189,22 @@ zfs_uio_iov_iter_init(zfs_uio_t *uio, struct iov_iter *iter, offset_t offset,
 	memset(&uio->uio_dio, 0, sizeof (zfs_uio_dio_t));
 }
 
+/*
+ * Rewind an iter-backed uio's underlying iov_iter by nbytes.  zfs_uiomove()
+ * advances the VFS iov_iter that uio_iter points at; a caller that saves and
+ * restores the zfs_uio_t struct on error (e.g. zfs_write_async()) restores
+ * only the scalar resid/offset, not the iterator position, since both structs
+ * share one iov_iter.  Use this to undo the advance before handing the
+ * iterator to a fallback path.  Only valid for UIO_ITER uios.
+ */
+static inline void
+zfs_uio_iov_iter_revert(zfs_uio_t *uio, size_t nbytes)
+{
+	ASSERT3S(uio->uio_segflg, ==, UIO_ITER);
+	if (nbytes > 0)
+		iov_iter_revert(uio->uio_iter, nbytes);
+}
+
 #if defined(HAVE_ITER_IOV)
 #define	zfs_uio_iter_iov(iter)	iter_iov((iter))
 #else

include/os/linux/zfs/sys/dmu_direct_os.h

@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: CDDL-1.0
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or https://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ * Copyright 2026, tiehexue <tiehexue@hotmail.com>. All rights reserved.
+ *
+ */
+
+#ifndef	_SYS_DMU_OS_H
+#define	_SYS_DMU_OS_H
+
+#include <sys/dmu.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+/*
+ * Async Direct I/O completion callback type (shared by read and write).
+ */
+typedef void (dmu_abd_done_func_t)(void *arg, int error);
+
+/*
+ * Async Direct I/O read.  Submits reads via the ZIO pipeline and returns
+ * immediately.  The completion callback fires from ZIO taskq context when
+ * all reads finish.  Caller retains ownership of 'data' until callback.
+ */
+int dmu_read_abd_async(dnode_t *dn, uint64_t offset, uint64_t size,
+    abd_t *data, dmu_flags_t flags,
+    dmu_abd_done_func_t *done, void *done_arg);
+
+/*
+ * Async Direct I/O write.  Submits writes via the ZIO pipeline and returns
+ * immediately.  The completion callback fires from ZIO taskq context when
+ * all writes finish.  Caller retains ownership of 'data' until callback
+ * and must commit the transaction (tx) from the callback.
+ */
+int dmu_write_abd_async(dnode_t *dn, uint64_t offset, uint64_t size,
+    abd_t *data, dmu_flags_t flags, dmu_tx_t *tx,
+    dmu_abd_done_func_t *done, void *done_arg);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _SYS_DMU_OS_H */

include/os/linux/zfs/sys/zfs_vfsops_os.h

@@ -103,6 +103,9 @@ struct zfsvfs {
 	boolean_t	z_use_hold;	/* held via dmu_objset_hold */
 	rrmlock_t	z_teardown_lock;
 	krwlock_t	z_teardown_inactive_lock;
+	kmutex_t	z_async_dio_lock; /* protects z_async_dio_inflight */
+	kcondvar_t	z_async_dio_cv;	/* signals drain to teardown */
+	uint64_t	z_async_dio_inflight; /* submitted, not yet completed */
 	list_t		z_all_znodes;	/* all znodes in the fs */
 	unsigned long	z_rollback_time; /* last online rollback time */
 	unsigned long	z_snap_defer_time; /* last snapshot unmount deferral */

include/os/linux/zfs/sys/zfs_vnops_os.h

@@ -94,6 +94,14 @@ extern int zfs_rename_idmap(znode_t *sdzp, char *snm, znode_t *tdzp,
     zidmap_t *idmap);
 extern int zfs_symlink_idmap(znode_t *dzp, char *name, vattr_t *vap,
     char *link, znode_t **zpp, cred_t *cr, int flags, zidmap_t *idmap);
+/* async Direct I/O */
+struct kiocb;
+extern int zfs_read_async(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr,
+    struct kiocb *kiocb);
+extern int zfs_write_async(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr,
+    struct kiocb *kiocb);
+extern void zfs_async_dio_init(void);
+extern void zfs_async_dio_fini(void);
 
 #ifdef	__cplusplus
 }

include/sys/dmu_impl.h

@@ -271,6 +271,19 @@ int dmu_write_direct(zio_t *, dmu_buf_impl_t *, abd_t *, dmu_tx_t *);
 int dmu_read_abd(dnode_t *, uint64_t, uint64_t, abd_t *, dmu_flags_t);
 int dmu_write_abd(dnode_t *, uint64_t, uint64_t, abd_t *, dmu_flags_t,
     dmu_tx_t *);
+
+/*
+ * Shared helpers used by both synchronous and asynchronous DMU I/O paths.
+ * These dispatch per-dbuf ZIOs into a caller-provided root zio; the caller
+ * decides whether to zio_wait() (sync) or zio_nowait() + callback (async).
+ */
+int dmu_read_abd_dispatch(zio_t *rio, dnode_t *dn, uint64_t offset,
+    uint64_t size, abd_t *data, dmu_flags_t flags,
+    dmu_buf_t **dbp, int numbufs);
+int dmu_write_abd_dispatch(zio_t *pio, dnode_t *dn, uint64_t offset,
+    uint64_t size, abd_t *data, dmu_flags_t flags, dmu_tx_t *tx,
+    dmu_buf_t **dbp, int numbufs);
+
 #if defined(_KERNEL)
 int dmu_read_uio_direct(dnode_t *, zfs_uio_t *, uint64_t, dmu_flags_t);
 int dmu_write_uio_direct(dnode_t *, zfs_uio_t *, uint64_t, dmu_flags_t,

include/sys/rrwlock.h

@@ -108,6 +108,7 @@ boolean_t rrm_held(rrmlock_t *rrl, krw_t rw);
 
 #define	RRM_READ_HELD(x)	rrm_held(x, RW_READER)
 #define	RRM_WRITE_HELD(x)	rrm_held(x, RW_WRITER)
+
 #define	RRM_LOCK_HELD(x) \
 	(rrm_held(x, RW_WRITER) || rrm_held(x, RW_READER))
 

include/sys/zfs_vnops.h

@@ -31,9 +31,33 @@
 
 extern int zfs_bclone_enabled;
 
+/*
+ * Direct I/O tunables.  zfs_dio_enabled can be set to 0 to force all
+ * I/O through the ARC; zfs_dio_strict returns EINVAL for unaligned
+ * DIO instead of falling back.
+ */
+extern int zfs_dio_enabled;
+extern int zfs_dio_strict;
+
 extern int zfs_fsync(znode_t *, int, cred_t *);
 extern int zfs_read(znode_t *, zfs_uio_t *, int, cred_t *);
 extern int zfs_write(znode_t *, zfs_uio_t *, int, cred_t *);
+
+/*
+ * Direct I/O page-pinning setup.  Pins user pages for O_DIRECT reads,
+ * enforces alignment, and skips DIO for mmap'd or encrypted ranges.
+ * Returns 0 and sets UIO_DIRECT in uio->uio_extflg on success.
+ */
+extern int zfs_setup_direct(struct znode *, zfs_uio_t *, zfs_uio_rw_t, int *);
+
+/*
+ * Clear the SUID/SGID bits after a write by non-owner.
+ * Called from the async write completion path (zfs_vnops_os.c on Linux)
+ * as well as from the synchronous zfs_write().
+ */
+extern void zfs_clear_setid_bits_if_necessary(zfsvfs_t *, znode_t *, cred_t *,
+    uint64_t *, dmu_tx_t *);
+
 extern int zfs_holey(znode_t *, ulong_t, loff_t *);
 extern int zfs_access(znode_t *, int, int, cred_t *);
 extern int zfs_clone_range(znode_t *, uint64_t *, znode_t *, uint64_t *,

module/Kbuild.in

@@ -458,6 +458,7 @@ ZFS_OBJS := \
 
 ZFS_OBJS_OS := \
 	abd_os.o \
+	dmu_direct_os.o \
 	arc_os.o \
 	kasan_compat.o \
 	mmp_os.o \

module/os/linux/zfs/dmu_direct_os.c

@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: CDDL-1.0
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or https://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ * Copyright 2026, tiehexue <tiehexue@hotmail.com>. All rights reserved.
+ *
+ */
+
+/*
+ * Linux async Direct I/O variants for the DMU layer.
+ *
+ * dmu_read_abd_async() / dmu_write_abd_async() submit I/O via the ZIO
+ * pipeline and return immediately.  Completions fire from ZIO taskq
+ * context via caller-provided callbacks (dmu_abd_done_func_t).
+ *
+ * These are Linux-only because only the Linux VFS layer has async
+ * kiocb / -EIOCBQUEUED infrastructure that benefits from non-blocking
+ * DMU entry points.
+ */
+
+#include <sys/dmu.h>
+#include <sys/dmu_impl.h>
+#include <sys/dmu_direct_os.h>
+#include <sys/dbuf.h>
+#include <sys/dnode.h>
+#include <sys/zfs_context.h>
+#include <sys/zfs_racct.h>
+#include <sys/dsl_dataset.h>
+#include <sys/dmu_objset.h>
+
+/*
+ * Shared async state for dmu_read_abd_async() and dmu_write_abd_async().
+ */
+struct dmu_abd_async_state {
+	dmu_buf_t	**ds_dbp;
+	int		ds_numbufs;
+	dmu_abd_done_func_t *ds_done;
+	void		*ds_done_arg;
+};
+
+static void
+dmu_abd_async_done(zio_t *zio)
+{
+	struct dmu_abd_async_state *ds = zio->io_private;
+	int error = zio->io_error;
+
+	dmu_buf_rele_array(ds->ds_dbp, ds->ds_numbufs, FTAG);
+	ds->ds_done(ds->ds_done_arg, error);
+	kmem_free(ds, sizeof (*ds));
+}
+
+/*
+ * Asynchronous variant of dmu_read_abd().  Uses the shared
+ * dmu_read_abd_dispatch() helper (common code) for the per-dbuf ZIO
+ * submission loop; only the async plumbing (root zio callback, state
+ * allocation, zio_nowait) lives here in the Linux-specific layer.
+ */
+int
+dmu_read_abd_async(dnode_t *dn, uint64_t offset, uint64_t size,
+    abd_t *data, dmu_flags_t flags,
+    dmu_abd_done_func_t *done, void *done_arg)
+{
+	spa_t *spa = dn->dn_objset->os_spa;
+	dmu_buf_t **dbp;
+	int numbufs, err;
+
+	ASSERT(flags & DMU_DIRECTIO);
+	ASSERT3P(done, !=, NULL);
+
+	err = dmu_buf_hold_array_by_dnode(dn, offset,
+	    size, B_FALSE, FTAG, &numbufs, &dbp, flags);
+	if (err)
+		return (err);
+
+	struct dmu_abd_async_state *ds =
+	    kmem_alloc(sizeof (*ds), KM_SLEEP);
+	ds->ds_dbp = dbp;
+	ds->ds_numbufs = numbufs;
+	ds->ds_done = done;
+	ds->ds_done_arg = done_arg;
+
+	zio_t *rio = zio_root(spa, dmu_abd_async_done, ds,
+	    ZIO_FLAG_CANFAIL);
+
+	err = dmu_read_abd_dispatch(rio, dn, offset, size, data, flags,
+	    dbp, numbufs);
+
+	/*
+	 * Dispatch the root zio.  On error, dmu_read_abd_dispatch() has
+	 * already set rio->io_error; the async done callback will release
+	 * dbp and signal the caller.
+	 */
+	zio_nowait(rio);
+	return (0);
+}
+
+
+
+/*
+ * Asynchronous variant of dmu_write_abd().  Uses the shared
+ * dmu_write_abd_dispatch() helper (common code) for the per-dbuf
+ * ZIO submission loop; only the async plumbing lives here.
+ */
+int
+dmu_write_abd_async(dnode_t *dn, uint64_t offset, uint64_t size,
+    abd_t *data, dmu_flags_t flags, dmu_tx_t *tx,
+    dmu_abd_done_func_t *done, void *done_arg)
+{
+	spa_t *spa = dn->dn_objset->os_spa;
+	dmu_buf_t **dbp;
+	int numbufs, err;
+
+	ASSERT(flags & DMU_DIRECTIO);
+	ASSERT3P(done, !=, NULL);
+
+	err = dmu_buf_hold_array_by_dnode(dn, offset,
+	    size, B_FALSE, FTAG, &numbufs, &dbp, flags);
+	if (err)
+		return (err);
+
+	struct dmu_abd_async_state *ds =
+	    kmem_alloc(sizeof (*ds), KM_SLEEP);
+	ds->ds_dbp = dbp;
+	ds->ds_numbufs = numbufs;
+	ds->ds_done = done;
+	ds->ds_done_arg = done_arg;
+
+	zio_t *pio = zio_root(spa, dmu_abd_async_done, ds,
+	    ZIO_FLAG_CANFAIL);
+
+	err = dmu_write_abd_dispatch(pio, dn, offset, size, data, flags, tx,
+	    dbp, numbufs);
+
+	if (err != 0)
+		pio->io_error = err;
+
+	zio_nowait(pio);
+	return (0);
+}