zio_crypt: establish platform interface; rework common code to use it

include/Makefile.am

@@ -146,6 +146,7 @@ COMMON_H = \
 	sys/zio_checksum.h \
 	sys/zio_compress.h \
 	sys/zio_crypt.h \
+	sys/zio_crypt_os_icp.h \
 	sys/zio_impl.h \
 	sys/zrlock.h \
 	sys/zthr.h \

include/os/freebsd/Makefile.am

@@ -93,5 +93,6 @@ noinst_HEADERS = \
 	%D%/zfs/sys/zfs_vfsops_os.h \
 	%D%/zfs/sys/zfs_vnops_os.h \
 	%D%/zfs/sys/zfs_znode_impl.h \
+	%D%/zfs/sys/zio_crypt_os.h \
 	%D%/zfs/sys/zpl.h
 

include/os/freebsd/spl/sys/uio.h

@@ -65,6 +65,7 @@ typedef struct zfs_uio {
 #define	zfs_uio_segflg(u)	GET_UIO_STRUCT(u)->uio_segflg
 #define	zfs_uio_offset(u)	GET_UIO_STRUCT(u)->uio_offset
 #define	zfs_uio_resid(u)	GET_UIO_STRUCT(u)->uio_resid
+#define	zfs_uio_iov(u)		GET_UIO_STRUCT(u)->uio_iov
 #define	zfs_uio_iovcnt(u)	GET_UIO_STRUCT(u)->uio_iovcnt
 #define	zfs_uio_iovlen(u, idx)	GET_UIO_STRUCT(u)->uio_iov[(idx)].iov_len
 #define	zfs_uio_iovbase(u, idx)	GET_UIO_STRUCT(u)->uio_iov[(idx)].iov_base

include/os/freebsd/zfs/sys/zio_crypt_os.h

@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: CDDL-1.0
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright (c) 2017, Datto, Inc. All rights reserved.
+ * Copyright (c) 2026, TrueNAS.
+ */
+
+#ifndef	_SYS_ZIO_CRYPT_OS_H
+#define	_SYS_ZIO_CRYPT_OS_H
+
+#include <sys/freebsd_crypto.h>
+
+typedef struct zio_crypt_session {
+	freebsd_crypt_session_t	zs_sess;
+} zio_crypt_session_t;
+
+typedef struct zio_crypt_hmac {
+	struct hmac_ctx		zh_ctx;
+} zio_crypt_hmac_t;
+
+#endif

include/os/linux/Makefile.am

@@ -49,6 +49,7 @@ kernel_sys_HEADERS = \
 	%D%/zfs/sys/zfs_vfsops_os.h \
 	%D%/zfs/sys/zfs_vnops_os.h \
 	%D%/zfs/sys/zfs_znode_impl.h \
+	%D%/zfs/sys/zio_crypt_os.h \
 	%D%/zfs/sys/zpl.h
 
 kernel_spl_rpcdir = $(kerneldir)/spl/rpc

include/os/linux/spl/sys/uio.h

@@ -69,8 +69,8 @@ typedef struct {
 
 typedef struct zfs_uio {
 	union {
-		const struct iovec	*uio_iov;
-		const struct bio_vec	*uio_bvec;
+		struct iovec		*uio_iov;
+		struct bio_vec		*uio_bvec;
 		struct iov_iter		*uio_iter;
 	};
 	int		uio_iovcnt;	/* Number of iovecs */
@@ -91,6 +91,7 @@ typedef struct zfs_uio {
 #define	zfs_uio_segflg(u)		(u)->uio_segflg
 #define	zfs_uio_offset(u)		(u)->uio_loffset
 #define	zfs_uio_resid(u)		(u)->uio_resid
+#define	zfs_uio_iov(u)			(u)->uio_iov
 #define	zfs_uio_iovcnt(u)		(u)->uio_iovcnt
 #define	zfs_uio_iovlen(u, idx)		(u)->uio_iov[(idx)].iov_len
 #define	zfs_uio_iovbase(u, idx)		(u)->uio_iov[(idx)].iov_base
@@ -122,7 +123,7 @@ zfs_uio_advance(zfs_uio_t *uio, ssize_t size)
 }
 
 static inline void
-zfs_uio_iovec_init(zfs_uio_t *uio, const struct iovec *iov,
+zfs_uio_iovec_init(zfs_uio_t *uio, struct iovec *iov,
     unsigned long nr_segs, offset_t offset, zfs_uio_seg_t seg, ssize_t resid,
     size_t skip)
 {

include/os/linux/zfs/sys/zio_crypt_os.h

@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: CDDL-1.0
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright (c) 2017, Datto, Inc. All rights reserved.
+ * Copyright (c) 2026, TrueNAS.
+ */
+
+#ifndef	_SYS_ZIO_CRYPT_OS_H
+#define	_SYS_ZIO_CRYPT_OS_H
+
+/* Linux always uses the ICP backend. */
+#include <sys/zio_crypt_os_icp.h>
+
+#endif

include/sys/zio_crypt.h

@@ -16,30 +16,22 @@
 
 /*
  * Copyright (c) 2017, Datto, Inc. All rights reserved.
+ * Copyright (c) 2026, TrueNAS.
  */
 
 #ifndef	_SYS_ZIO_CRYPT_H
 #define	_SYS_ZIO_CRYPT_H
 
 #include <sys/dmu.h>
-#include <sys/zfs_refcount.h>
-#if defined(__FreeBSD__) && defined(_KERNEL)
-#include <sys/freebsd_crypto.h>
-#else
-#include <sys/crypto/api.h>
-#endif /* __FreeBSD__ */
-#include <sys/nvpair.h>
-#include <sys/avl.h>
 #include <sys/zio.h>
-
-/* forward declarations */
-struct zbookmark_phys;
+#include <sys/zio_crypt_os.h>
 
 #define	WRAPPING_KEY_LEN	32
 #define	WRAPPING_IV_LEN		ZIO_DATA_IV_LEN
 #define	WRAPPING_MAC_LEN	ZIO_DATA_MAC_LEN
 #define	MASTER_KEY_MAX_LEN	32
 #define	SHA512_HMAC_KEYLEN	64
+#define	SHA512_HMAC_LEN		64
 
 #define	ZIO_CRYPT_KEY_CURRENT_VERSION	1ULL
 
@@ -51,16 +43,9 @@ typedef enum zio_crypt_type {
 
 /* table of supported crypto algorithms, modes and keylengths. */
 typedef struct zio_crypt_info {
-	/* mechanism name, needed by ICP */
-#if defined(__FreeBSD__) && defined(_KERNEL)
-	/*
-	 * I've deliberately used a different name here, to catch
-	 * ICP-using code.
-	 */
-	const char	*ci_algname;
-#else
-	crypto_mech_name_t ci_mechname;
-#endif
+	/* mechanism/algorithm name for backend to select implementation */
+	const char *ci_mechname;
+
 	/* cipher mode type (GCM, CCM) */
 	zio_crypt_type_t ci_crypt_type;
 
@@ -99,22 +84,17 @@ typedef struct zio_crypt_key {
 	/* count of how many times the current salt has been used */
 	uint64_t zk_salt_count;
 
-	/* illumos crypto api current encryption key */
+	/* current raw encryption key for backend */
 	crypto_key_t zk_current_key;
 
-#if defined(__FreeBSD__) && defined(_KERNEL)
-	/* Session for current encryption key.  Must always be set */
-	freebsd_crypt_session_t	zk_session;
-#else
-	/* template of current encryption key for illumos crypto api */
-	crypto_ctx_template_t zk_current_tmpl;
-#endif
+	/* backend template (session) for current encryption key */
+	zio_crypt_session_t zk_current_sess;
 
-	/* illumos crypto api current hmac key */
+	/* current raw hmac key for backend */
 	crypto_key_t zk_hmac_key;
 
-	/* template of hmac key for illumos crypto api */
-	crypto_ctx_template_t zk_hmac_tmpl;
+	/* backend template (session) for current hmac key */
+	zio_crypt_session_t zk_hmac_sess;
 
 	/* lock for changing the salt and dependent values */
 	krwlock_t zk_salt_lock;
@@ -158,4 +138,58 @@ int zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key,
     uint8_t *mac, uint_t datalen, abd_t *pabd, abd_t *cabd,
     boolean_t *no_crypt);
 
+/*
+ * Platform/backend interface to an arbitrary crypto suite.
+ */
+
+/*
+ * A key must be opened before use, so the backend can initialise the wanted
+ * algorithm and set up the backend crypto suite/hardware.
+ *
+ * Reopen should be done after any of the keys internal properties change,
+ * eg the salt is changed. It is logically equivalent to close+open, but the
+ * backend may be able to do it more efficiently.
+ */
+int zio_crypt_key_open_os(zio_crypt_key_t *key, const zio_crypt_info_t *ci);
+void zio_crypt_key_close_os(zio_crypt_key_t *key);
+int zio_crypt_key_reopen_os(zio_crypt_key_t *key, const zio_crypt_info_t *ci);
+
+/*
+ * Initialise/free a pair of UIOs with space for iovcnt data elements. This
+ * will allocate/free zfs_uio_iov(u1)/zfs_uio_iov(u2) with at least iovcnt
+ * elements. All other uio fields are private to the backend.
+ *
+ * The backend may allocate more elements than requested; the first
+ * one available for caller data is returned in *idx, and the caller should
+ * not try to use beyond the *idx+iovcnt-1 element.
+ */
+int zio_crypt_uios_init_os(zfs_uio_t *u1, zfs_uio_t *u2, int iovcnt, int *idx);
+void zio_crypt_uios_fini_os(zfs_uio_t *u1, zfs_uio_t *u2);
+
+/* Low-level encrypt/decrypt. See zio_do_crypt_data(). */
+int zio_encrypt_os(const zio_crypt_info_t *ci,
+    crypto_key_t *key, zio_crypt_session_t *sess,
+    zfs_uio_t *plaintext, zfs_uio_t *ciphertext, size_t datalen,
+    const uint8_t iv[ZIO_DATA_IV_LEN], const uint8_t *ad, size_t adlen,
+    uint8_t mac[ZIO_DATA_MAC_LEN]);
+int zio_decrypt_os(const zio_crypt_info_t *ci,
+    crypto_key_t *key, zio_crypt_session_t *sess,
+    zfs_uio_t *ciphertext, zfs_uio_t *plaintext, size_t datalen,
+    const uint8_t iv[ZIO_DATA_IV_LEN], const uint8_t *ad, size_t adlen,
+    uint8_t mac[ZIO_DATA_MAC_LEN]);
+
+/* Generate SHA512-HMAC digest of data using the given HMAC key. */
+int zio_crypt_hmac_os(zio_crypt_key_t *key, const uint8_t *data,
+    size_t datalen, uint8_t digest[SHA512_HMAC_LEN]);
+
+/*
+ * Generate SHA512-HMAC digest using given key. Data is passed incrementally,
+ * and the final result generated at the end.
+ */
+int zio_crypt_hmac_init_os(zio_crypt_hmac_t *hmac, zio_crypt_key_t *key);
+int zio_crypt_hmac_update_os(zio_crypt_hmac_t *hmac, const uint8_t *data,
+    size_t datalen);
+int zio_crypt_hmac_final_os(zio_crypt_hmac_t *hmac,
+	uint8_t digest[SHA512_HMAC_LEN]);
+
 #endif

include/sys/zio_crypt_os_icp.h

@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: CDDL-1.0
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright (c) 2017, Datto, Inc. All rights reserved.
+ * Copyright (c) 2026, TrueNAS.
+ */
+
+#ifndef	_SYS_ZIO_CRYPT_OS_ICP_H
+#define	_SYS_ZIO_CRYPT_OS_ICP_H
+
+#include <sys/crypto/api.h>
+
+typedef struct zio_crypt_session {
+	crypto_ctx_template_t	zs_tmpl;
+} zio_crypt_session_t;
+
+typedef struct zio_crypt_hmac {
+	crypto_context_t	zh_ctx;
+} zio_crypt_hmac_t;
+
+#endif

lib/libspl/include/sys/uio.h

@@ -79,6 +79,7 @@ typedef struct zfs_uio {
 #define	zfs_uio_segflg(uio)		(uio)->uio_segflg
 #define	zfs_uio_offset(uio)		(uio)->uio_loffset
 #define	zfs_uio_resid(uio)		(uio)->uio_resid
+#define	zfs_uio_iov(uio)		(uio)->uio_iov
 #define	zfs_uio_iovcnt(uio)		(uio)->uio_iovcnt
 #define	zfs_uio_iovlen(uio, idx)	(uio)->uio_iov[(idx)].iov_len
 #define	zfs_uio_iovbase(uio, idx)	(uio)->uio_iov[(idx)].iov_base

lib/libzpool/Makefile.am

@@ -54,8 +54,6 @@ ZCP_SOURCES = \
 	module/zfs/zcp_synctask.c
 
 nodist_libzpool_la_SOURCES = \
-	module/os/linux/zfs/zio_crypt.c \
-	\
 	module/zcommon/cityhash.c \
 	module/zcommon/simd_stat.c \
 	module/zcommon/zfeature_common.c \
@@ -194,6 +192,8 @@ nodist_libzpool_la_SOURCES = \
 	module/zfs/zio.c \
 	module/zfs/zio_checksum.c \
 	module/zfs/zio_compress.c \
+	module/zfs/zio_crypt.c \
+	module/zfs/zio_crypt_os_icp.c \
 	module/zfs/zio_inject.c \
 	module/zfs/zle.c \
 	module/zfs/zrlock.c \

lib/libzpool/include/Makefile.am

@@ -6,4 +6,5 @@ libzpool_sys_HEADERS = \
 	%D%/sys/trace_zfs.h \
 	%D%/sys/zfs_bootenv_os.h \
 	%D%/sys/zfs_context_os.h \
-	%D%/sys/zfs_debug_os.h
+	%D%/sys/zfs_debug_os.h \
+	%D%/sys/zio_crypt_os.h

lib/libzpool/include/sys/zio_crypt_os.h

@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: CDDL-1.0
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright (c) 2017, Datto, Inc. All rights reserved.
+ * Copyright (c) 2026, TrueNAS.
+ */
+
+#ifndef	_SYS_ZIO_CRYPT_OS_H
+#define	_SYS_ZIO_CRYPT_OS_H
+
+/* libzpool always uses the ICP backend. */
+#include <sys/zio_crypt_os_icp.h>
+
+#endif

module/Kbuild.in

@@ -449,6 +449,8 @@ ZFS_OBJS := \
 	zio.o \
 	zio_checksum.o \
 	zio_compress.o \
+	zio_crypt.o \
+	zio_crypt_os_icp.o \
 	zio_inject.o \
 	zle.o \
 	zrlock.o \
@@ -482,7 +484,6 @@ ZFS_OBJS_OS := \
 	zfs_vfsops.o \
 	zfs_vnops_os.o \
 	zfs_znode_os.o \
-	zio_crypt.o \
 	zpl_ctldir.o \
 	zpl_export.o \
 	zpl_file.o \

module/Makefile.bsd

@@ -231,7 +231,7 @@ SRCS+=	abd_os.c \
 	zfs_vfsops.c \
 	zfs_vnops_os.c \
 	zfs_znode_os.c \
-	zio_crypt.c \
+	zio_crypt_os.c \
 	zvol_os.c
 
 #zcommon
@@ -381,6 +381,7 @@ SRCS+=	abd.c \
 	zio.c \
 	zio_checksum.c \
 	zio_compress.c \
+	zio_crypt.c \
 	zio_inject.c \
 	zle.c \
 	zrlock.c \