diff --git a/include/Makefile.am b/include/Makefile.am index 76ae39f51f12..05b583ee803d 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -146,6 +146,7 @@ COMMON_H = \ sys/zio_checksum.h \ sys/zio_compress.h \ sys/zio_crypt.h \ + sys/zio_crypt_os_icp.h \ sys/zio_impl.h \ sys/zrlock.h \ sys/zthr.h \ diff --git a/include/os/freebsd/Makefile.am b/include/os/freebsd/Makefile.am index a32c08bff00b..c2d86823bffe 100644 --- a/include/os/freebsd/Makefile.am +++ b/include/os/freebsd/Makefile.am @@ -93,5 +93,6 @@ noinst_HEADERS = \ %D%/zfs/sys/zfs_vfsops_os.h \ %D%/zfs/sys/zfs_vnops_os.h \ %D%/zfs/sys/zfs_znode_impl.h \ + %D%/zfs/sys/zio_crypt_os.h \ %D%/zfs/sys/zpl.h diff --git a/include/os/freebsd/spl/sys/uio.h b/include/os/freebsd/spl/sys/uio.h index 528de695525a..6b33a7e71000 100644 --- a/include/os/freebsd/spl/sys/uio.h +++ b/include/os/freebsd/spl/sys/uio.h @@ -65,6 +65,7 @@ typedef struct zfs_uio { #define zfs_uio_segflg(u) GET_UIO_STRUCT(u)->uio_segflg #define zfs_uio_offset(u) GET_UIO_STRUCT(u)->uio_offset #define zfs_uio_resid(u) GET_UIO_STRUCT(u)->uio_resid +#define zfs_uio_iov(u) GET_UIO_STRUCT(u)->uio_iov #define zfs_uio_iovcnt(u) GET_UIO_STRUCT(u)->uio_iovcnt #define zfs_uio_iovlen(u, idx) GET_UIO_STRUCT(u)->uio_iov[(idx)].iov_len #define zfs_uio_iovbase(u, idx) GET_UIO_STRUCT(u)->uio_iov[(idx)].iov_base diff --git a/include/os/freebsd/zfs/sys/zio_crypt_os.h b/include/os/freebsd/zfs/sys/zio_crypt_os.h new file mode 100644 index 000000000000..cfb25c5cf4b2 --- /dev/null +++ b/include/os/freebsd/zfs/sys/zio_crypt_os.h @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + */ + +/* + * Copyright (c) 2017, Datto, Inc. All rights reserved. + * Copyright (c) 2026, TrueNAS. + */ + +#ifndef _SYS_ZIO_CRYPT_OS_H +#define _SYS_ZIO_CRYPT_OS_H + +#include + +typedef struct zio_crypt_session { + freebsd_crypt_session_t zs_sess; +} zio_crypt_session_t; + +typedef struct zio_crypt_hmac { + struct hmac_ctx zh_ctx; +} zio_crypt_hmac_t; + +#endif diff --git a/include/os/linux/Makefile.am b/include/os/linux/Makefile.am index 3844bf87e9ce..7121741bee5e 100644 --- a/include/os/linux/Makefile.am +++ b/include/os/linux/Makefile.am @@ -50,6 +50,7 @@ kernel_sys_HEADERS = \ %D%/zfs/sys/zfs_vfsops_os.h \ %D%/zfs/sys/zfs_vnops_os.h \ %D%/zfs/sys/zfs_znode_impl.h \ + %D%/zfs/sys/zio_crypt_os.h \ %D%/zfs/sys/zpl.h kernel_spl_rpcdir = $(kerneldir)/spl/rpc diff --git a/include/os/linux/spl/sys/uio.h b/include/os/linux/spl/sys/uio.h index 26c2c387caa3..0efad1b4a14e 100644 --- a/include/os/linux/spl/sys/uio.h +++ b/include/os/linux/spl/sys/uio.h @@ -69,8 +69,8 @@ typedef struct { typedef struct zfs_uio { union { - const struct iovec *uio_iov; - const struct bio_vec *uio_bvec; + struct iovec *uio_iov; + struct bio_vec *uio_bvec; struct iov_iter *uio_iter; }; int uio_iovcnt; /* Number of iovecs */ @@ -91,6 +91,7 @@ typedef struct zfs_uio { #define zfs_uio_segflg(u) (u)->uio_segflg #define zfs_uio_offset(u) (u)->uio_loffset #define zfs_uio_resid(u) (u)->uio_resid +#define zfs_uio_iov(u) (u)->uio_iov #define zfs_uio_iovcnt(u) (u)->uio_iovcnt #define zfs_uio_iovlen(u, idx) (u)->uio_iov[(idx)].iov_len #define zfs_uio_iovbase(u, idx) (u)->uio_iov[(idx)].iov_base @@ -122,7 +123,7 @@ zfs_uio_advance(zfs_uio_t *uio, ssize_t size) } static inline void -zfs_uio_iovec_init(zfs_uio_t *uio, const struct iovec *iov, +zfs_uio_iovec_init(zfs_uio_t *uio, struct iovec *iov, unsigned long nr_segs, offset_t offset, zfs_uio_seg_t seg, ssize_t resid, size_t skip) { diff --git a/include/os/linux/zfs/sys/zio_crypt_os.h b/include/os/linux/zfs/sys/zio_crypt_os.h new file mode 100644 index 000000000000..0729931aecfb --- /dev/null +++ b/include/os/linux/zfs/sys/zio_crypt_os.h @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + */ + +/* + * Copyright (c) 2017, Datto, Inc. All rights reserved. + * Copyright (c) 2026, TrueNAS. + */ + +#ifndef _SYS_ZIO_CRYPT_OS_H +#define _SYS_ZIO_CRYPT_OS_H + +/* Linux always uses the ICP backend. */ +#include + +#endif diff --git a/include/sys/zio_crypt.h b/include/sys/zio_crypt.h index 30ba198085df..d2ae4580d0f3 100644 --- a/include/sys/zio_crypt.h +++ b/include/sys/zio_crypt.h @@ -16,30 +16,22 @@ /* * Copyright (c) 2017, Datto, Inc. All rights reserved. + * Copyright (c) 2026, TrueNAS. */ #ifndef _SYS_ZIO_CRYPT_H #define _SYS_ZIO_CRYPT_H #include -#include -#if defined(__FreeBSD__) && defined(_KERNEL) -#include -#else -#include -#endif /* __FreeBSD__ */ -#include -#include #include - -/* forward declarations */ -struct zbookmark_phys; +#include #define WRAPPING_KEY_LEN 32 #define WRAPPING_IV_LEN ZIO_DATA_IV_LEN #define WRAPPING_MAC_LEN ZIO_DATA_MAC_LEN #define MASTER_KEY_MAX_LEN 32 #define SHA512_HMAC_KEYLEN 64 +#define SHA512_HMAC_LEN 64 #define ZIO_CRYPT_KEY_CURRENT_VERSION 1ULL @@ -51,16 +43,9 @@ typedef enum zio_crypt_type { /* table of supported crypto algorithms, modes and keylengths. */ typedef struct zio_crypt_info { - /* mechanism name, needed by ICP */ -#if defined(__FreeBSD__) && defined(_KERNEL) - /* - * I've deliberately used a different name here, to catch - * ICP-using code. - */ - const char *ci_algname; -#else - crypto_mech_name_t ci_mechname; -#endif + /* mechanism/algorithm name for backend to select implementation */ + const char *ci_mechname; + /* cipher mode type (GCM, CCM) */ zio_crypt_type_t ci_crypt_type; @@ -99,22 +84,17 @@ typedef struct zio_crypt_key { /* count of how many times the current salt has been used */ uint64_t zk_salt_count; - /* illumos crypto api current encryption key */ + /* current raw encryption key for backend */ crypto_key_t zk_current_key; -#if defined(__FreeBSD__) && defined(_KERNEL) - /* Session for current encryption key. Must always be set */ - freebsd_crypt_session_t zk_session; -#else - /* template of current encryption key for illumos crypto api */ - crypto_ctx_template_t zk_current_tmpl; -#endif + /* backend template (session) for current encryption key */ + zio_crypt_session_t zk_current_sess; - /* illumos crypto api current hmac key */ + /* current raw hmac key for backend */ crypto_key_t zk_hmac_key; - /* template of hmac key for illumos crypto api */ - crypto_ctx_template_t zk_hmac_tmpl; + /* backend template (session) for current hmac key */ + zio_crypt_session_t zk_hmac_sess; /* lock for changing the salt and dependent values */ krwlock_t zk_salt_lock; @@ -158,4 +138,58 @@ int zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, uint8_t *mac, uint_t datalen, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt); +/* + * Platform/backend interface to an arbitrary crypto suite. + */ + +/* + * A key must be opened before use, so the backend can initialise the wanted + * algorithm and set up the backend crypto suite/hardware. + * + * Reopen should be done after any of the keys internal properties change, + * eg the salt is changed. It is logically equivalent to close+open, but the + * backend may be able to do it more efficiently. + */ +int zio_crypt_key_open_os(zio_crypt_key_t *key, const zio_crypt_info_t *ci); +void zio_crypt_key_close_os(zio_crypt_key_t *key); +int zio_crypt_key_reopen_os(zio_crypt_key_t *key, const zio_crypt_info_t *ci); + +/* + * Initialise/free a pair of UIOs with space for iovcnt data elements. This + * will allocate/free zfs_uio_iov(u1)/zfs_uio_iov(u2) with at least iovcnt + * elements. All other uio fields are private to the backend. + * + * The backend may allocate more elements than requested; the first + * one available for caller data is returned in *idx, and the caller should + * not try to use beyond the *idx+iovcnt-1 element. + */ +int zio_crypt_uios_init_os(zfs_uio_t *u1, zfs_uio_t *u2, int iovcnt, int *idx); +void zio_crypt_uios_fini_os(zfs_uio_t *u1, zfs_uio_t *u2); + +/* Low-level encrypt/decrypt. See zio_do_crypt_data(). */ +int zio_encrypt_os(const zio_crypt_info_t *ci, + crypto_key_t *key, zio_crypt_session_t *sess, + zfs_uio_t *plaintext, zfs_uio_t *ciphertext, size_t datalen, + const uint8_t iv[ZIO_DATA_IV_LEN], const uint8_t *ad, size_t adlen, + uint8_t mac[ZIO_DATA_MAC_LEN]); +int zio_decrypt_os(const zio_crypt_info_t *ci, + crypto_key_t *key, zio_crypt_session_t *sess, + zfs_uio_t *ciphertext, zfs_uio_t *plaintext, size_t datalen, + const uint8_t iv[ZIO_DATA_IV_LEN], const uint8_t *ad, size_t adlen, + uint8_t mac[ZIO_DATA_MAC_LEN]); + +/* Generate SHA512-HMAC digest of data using the given HMAC key. */ +int zio_crypt_hmac_os(zio_crypt_key_t *key, const uint8_t *data, + size_t datalen, uint8_t digest[SHA512_HMAC_LEN]); + +/* + * Generate SHA512-HMAC digest using given key. Data is passed incrementally, + * and the final result generated at the end. + */ +int zio_crypt_hmac_init_os(zio_crypt_hmac_t *hmac, zio_crypt_key_t *key); +int zio_crypt_hmac_update_os(zio_crypt_hmac_t *hmac, const uint8_t *data, + size_t datalen); +int zio_crypt_hmac_final_os(zio_crypt_hmac_t *hmac, + uint8_t digest[SHA512_HMAC_LEN]); + #endif diff --git a/include/sys/zio_crypt_os_icp.h b/include/sys/zio_crypt_os_icp.h new file mode 100644 index 000000000000..ed70ef7ea3f9 --- /dev/null +++ b/include/sys/zio_crypt_os_icp.h @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + */ + +/* + * Copyright (c) 2017, Datto, Inc. All rights reserved. + * Copyright (c) 2026, TrueNAS. + */ + +#ifndef _SYS_ZIO_CRYPT_OS_ICP_H +#define _SYS_ZIO_CRYPT_OS_ICP_H + +#include + +typedef struct zio_crypt_session { + crypto_ctx_template_t zs_tmpl; +} zio_crypt_session_t; + +typedef struct zio_crypt_hmac { + crypto_context_t zh_ctx; +} zio_crypt_hmac_t; + +#endif diff --git a/lib/libspl/include/sys/uio.h b/lib/libspl/include/sys/uio.h index 9ada482be000..08fab5e48175 100644 --- a/lib/libspl/include/sys/uio.h +++ b/lib/libspl/include/sys/uio.h @@ -79,6 +79,7 @@ typedef struct zfs_uio { #define zfs_uio_segflg(uio) (uio)->uio_segflg #define zfs_uio_offset(uio) (uio)->uio_loffset #define zfs_uio_resid(uio) (uio)->uio_resid +#define zfs_uio_iov(uio) (uio)->uio_iov #define zfs_uio_iovcnt(uio) (uio)->uio_iovcnt #define zfs_uio_iovlen(uio, idx) (uio)->uio_iov[(idx)].iov_len #define zfs_uio_iovbase(uio, idx) (uio)->uio_iov[(idx)].iov_base diff --git a/lib/libzpool/Makefile.am b/lib/libzpool/Makefile.am index a549f10891b5..ffae4701d119 100644 --- a/lib/libzpool/Makefile.am +++ b/lib/libzpool/Makefile.am @@ -54,8 +54,6 @@ ZCP_SOURCES = \ module/zfs/zcp_synctask.c nodist_libzpool_la_SOURCES = \ - module/os/linux/zfs/zio_crypt.c \ - \ module/zcommon/cityhash.c \ module/zcommon/simd_stat.c \ module/zcommon/zfeature_common.c \ @@ -194,6 +192,8 @@ nodist_libzpool_la_SOURCES = \ module/zfs/zio.c \ module/zfs/zio_checksum.c \ module/zfs/zio_compress.c \ + module/zfs/zio_crypt.c \ + module/zfs/zio_crypt_os_icp.c \ module/zfs/zio_inject.c \ module/zfs/zle.c \ module/zfs/zrlock.c \ diff --git a/lib/libzpool/include/Makefile.am b/lib/libzpool/include/Makefile.am index 53e07446c2e7..50a66b76d020 100644 --- a/lib/libzpool/include/Makefile.am +++ b/lib/libzpool/include/Makefile.am @@ -6,4 +6,5 @@ libzpool_sys_HEADERS = \ %D%/sys/trace_zfs.h \ %D%/sys/zfs_bootenv_os.h \ %D%/sys/zfs_context_os.h \ - %D%/sys/zfs_debug_os.h + %D%/sys/zfs_debug_os.h \ + %D%/sys/zio_crypt_os.h diff --git a/lib/libzpool/include/sys/zio_crypt_os.h b/lib/libzpool/include/sys/zio_crypt_os.h new file mode 100644 index 000000000000..c1b9f7825ade --- /dev/null +++ b/lib/libzpool/include/sys/zio_crypt_os.h @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + */ + +/* + * Copyright (c) 2017, Datto, Inc. All rights reserved. + * Copyright (c) 2026, TrueNAS. + */ + +#ifndef _SYS_ZIO_CRYPT_OS_H +#define _SYS_ZIO_CRYPT_OS_H + +/* libzpool always uses the ICP backend. */ +#include + +#endif diff --git a/module/Kbuild.in b/module/Kbuild.in index d6f8894f86ee..19d1fae3cf9d 100644 --- a/module/Kbuild.in +++ b/module/Kbuild.in @@ -449,6 +449,8 @@ ZFS_OBJS := \ zio.o \ zio_checksum.o \ zio_compress.o \ + zio_crypt.o \ + zio_crypt_os_icp.o \ zio_inject.o \ zle.o \ zrlock.o \ @@ -482,7 +484,6 @@ ZFS_OBJS_OS := \ zfs_vfsops.o \ zfs_vnops_os.o \ zfs_znode_os.o \ - zio_crypt.o \ zpl_ctldir.o \ zpl_export.o \ zpl_file.o \ diff --git a/module/Makefile.bsd b/module/Makefile.bsd index 436b090d9a00..f59d69b5e4e6 100644 --- a/module/Makefile.bsd +++ b/module/Makefile.bsd @@ -231,7 +231,7 @@ SRCS+= abd_os.c \ zfs_vfsops.c \ zfs_vnops_os.c \ zfs_znode_os.c \ - zio_crypt.c \ + zio_crypt_os.c \ zvol_os.c #zcommon @@ -381,6 +381,7 @@ SRCS+= abd.c \ zio.c \ zio_checksum.c \ zio_compress.c \ + zio_crypt.c \ zio_inject.c \ zle.c \ zrlock.c \ diff --git a/module/os/freebsd/zfs/crypto_os.c b/module/os/freebsd/zfs/crypto_os.c index a4bf3fb6490f..0e0a54f1ed9b 100644 --- a/module/os/freebsd/zfs/crypto_os.c +++ b/module/os/freebsd/zfs/crypto_os.c @@ -212,14 +212,12 @@ freebsd_crypt_uio_debug_log(boolean_t encrypt, size_t auth_len) { #ifdef FCRYPTO_DEBUG - struct cryptodesc *crd; - uint8_t *p = NULL; size_t total = 0; printf("%s(%s, %p, { %s, %d, %d, %s }, %p, { %p, %u }, " "%p, %u, %u)\n", __FUNCTION__, encrypt ? "encrypt" : "decrypt", input_sessionp, - c_info->ci_algname, c_info->ci_crypt_type, + c_info->ci_mechname, c_info->ci_crypt_type, (unsigned int)c_info->ci_keylen, c_info->ci_name, data_uio, key->ck_data, (unsigned int)key->ck_length, @@ -254,7 +252,7 @@ freebsd_crypt_newsession(freebsd_crypt_session_t *sessp, #ifdef FCRYPTO_DEBUG printf("%s(%p, { %s, %d, %d, %s }, { %p, %u })\n", __FUNCTION__, sessp, - c_info->ci_algname, c_info->ci_crypt_type, + c_info->ci_mechname, c_info->ci_crypt_type, (unsigned int)c_info->ci_keylen, c_info->ci_name, key->ck_data, (unsigned int)key->ck_length); printf("\tkey = { "); diff --git a/module/os/freebsd/zfs/zio_crypt_os.c b/module/os/freebsd/zfs/zio_crypt_os.c new file mode 100644 index 000000000000..e3d906d69796 --- /dev/null +++ b/module/os/freebsd/zfs/zio_crypt_os.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + */ + +/* + * Copyright (c) 2017, Datto, Inc. All rights reserved. + * Copyright (c) 2026, TrueNAS. + */ + +#include + +void +zio_crypt_key_close_os(zio_crypt_key_t *key) +{ + freebsd_crypt_freesession(&key->zk_current_sess.zs_sess); +} + +int +zio_crypt_key_open_os(zio_crypt_key_t *key, const zio_crypt_info_t *ci) +{ + return (freebsd_crypt_newsession(&key->zk_current_sess.zs_sess, ci, + &key->zk_current_key)); +} + +int +zio_crypt_key_reopen_os(zio_crypt_key_t *key, const zio_crypt_info_t *ci) +{ + freebsd_crypt_freesession(&key->zk_current_sess.zs_sess); + return (freebsd_crypt_newsession(&key->zk_current_sess.zs_sess, ci, + &key->zk_current_key)); +} + +int +zio_crypt_uios_init_os(zfs_uio_t *u1, zfs_uio_t *u2, int iovcnt, int *idx) +{ + /* + * One additional iovec at start for a single AAD buffer, and one + * at end for the MAC. + */ + iovcnt += 2; + + zfs_uio_init(u1, allocuio(iovcnt)); + zfs_uio_init(u2, allocuio(iovcnt)); + + zfs_uio_iovcnt(u1) = zfs_uio_iovcnt(u2) = iovcnt; + zfs_uio_segflg(u1) = zfs_uio_segflg(u2) = UIO_SYSSPACE; + + /* + * freebsd_crypt_uio() expects AD in slot 0, so first data iovec is + * slot 1. + */ + *idx = 1; + + return (0); +} + +void +zio_crypt_uios_fini_os(zfs_uio_t *u1, zfs_uio_t *u2) { + ASSERT3U(zfs_uio_iovcnt(u1), ==, zfs_uio_iovcnt(u2)); + ASSERT3U(zfs_uio_iovcnt(u1), >=, 2); + + freeuio(GET_UIO_STRUCT(u1)); + freeuio(GET_UIO_STRUCT(u2)); +} + +static int +zio_encrypt_decrypt_os_common(boolean_t do_encrypt, const zio_crypt_info_t *ci, + crypto_key_t *key, zio_crypt_session_t *sess, + zfs_uio_t *src, zfs_uio_t *dst, size_t datalen, + const uint8_t iv[ZIO_DATA_IV_LEN], const uint8_t *ad, size_t adlen, + uint8_t mac[ZIO_DATA_MAC_LEN]) +{ + ASSERT3U(zfs_uio_iovcnt(src), ==, zfs_uio_iovcnt(dst)); + ASSERT3U(zfs_uio_iovcnt(src), >=, 2); + + /* + * Our FreeBSD kernel crypto shim (crypto_os.c) currently only supports + * a single buffer for input and output. So we copy the input to the + * output here, and then work on the output buffers. + */ + for (int i = 1; i < zfs_uio_iovcnt(src)-1; i++) { + ASSERT3U(zfs_uio_iovlen(src, i), ==, + zfs_uio_iovlen(dst, i)); + memcpy(zfs_uio_iovbase(dst, i), + zfs_uio_iovbase(src, i), zfs_uio_iovlen(src, i)); + } + + /* Shim expects AAD in the first slot */ + zfs_uio_iovbase(dst, 0) = (void *)(uintptr_t)ad; + zfs_uio_iovlen(dst, 0) = adlen; + + /* And MAC in the last slot */ + zfs_uio_iovbase(dst, zfs_uio_iovcnt(dst)-1) = mac; + zfs_uio_iovlen(dst, zfs_uio_iovcnt(dst)-1) = ZIO_DATA_MAC_LEN; + + freebsd_crypt_session_t *fsess = sess != NULL ? &sess->zs_sess : NULL; + int ret = freebsd_crypt_uio(do_encrypt, fsess, + ci, dst, key, (uint8_t *)(uintptr_t)iv, datalen, adlen); + if (ret != 0) { +#ifdef FCRYPTO_DEBUG + printf("%s(%d): Returning error %s\n", + __FUNCTION__, __LINE__, do_encrypt ? "EIO" : "ECKSUM"); +#endif + ret = SET_ERROR(do_encrypt ? EIO : ECKSUM); + } + + return (ret); +} + +int +zio_encrypt_os(const zio_crypt_info_t *ci, + crypto_key_t *key, zio_crypt_session_t *sess, + zfs_uio_t *plaintext, zfs_uio_t *ciphertext, size_t datalen, + const uint8_t iv[ZIO_DATA_IV_LEN], const uint8_t *ad, size_t adlen, + uint8_t mac[ZIO_DATA_MAC_LEN]) +{ + return (zio_encrypt_decrypt_os_common(B_TRUE, ci, key, sess, + plaintext, ciphertext, datalen, iv, ad, adlen, mac)); +} + +int +zio_decrypt_os(const zio_crypt_info_t *ci, + crypto_key_t *key, zio_crypt_session_t *sess, + zfs_uio_t *ciphertext, zfs_uio_t *plaintext, size_t datalen, + const uint8_t iv[ZIO_DATA_IV_LEN], const uint8_t *ad, size_t adlen, + uint8_t mac[ZIO_DATA_MAC_LEN]) +{ + return (zio_encrypt_decrypt_os_common(B_FALSE, ci, key, sess, + ciphertext, plaintext, datalen, iv, ad, adlen, mac)); +} + +int +zio_crypt_hmac_os(zio_crypt_key_t *key, const uint8_t *data, size_t datalen, + uint8_t digest[SHA512_HMAC_LEN]) +{ + crypto_mac(&key->zk_hmac_key, data, datalen, digest, SHA512_HMAC_LEN); + return (0); +} + +int +zio_crypt_hmac_init_os(zio_crypt_hmac_t *hmac, zio_crypt_key_t *key) +{ + crypto_mac_init(&hmac->zh_ctx, &key->zk_hmac_key); + return (0); +} + +int +zio_crypt_hmac_update_os(zio_crypt_hmac_t *hmac, const uint8_t *data, + size_t datalen) +{ + crypto_mac_update(&hmac->zh_ctx, data, datalen); + return (0); +} + +int +zio_crypt_hmac_final_os(zio_crypt_hmac_t *hmac, + uint8_t digest[SHA512_HMAC_LEN]) +{ + crypto_mac_final(&hmac->zh_ctx, digest, SHA512_HMAC_LEN); + return (0); +} diff --git a/module/os/linux/zfs/zfs_uio.c b/module/os/linux/zfs/zfs_uio.c index bfce9e6b5202..ae721fc67614 100644 --- a/module/os/linux/zfs/zfs_uio.c +++ b/module/os/linux/zfs/zfs_uio.c @@ -65,7 +65,7 @@ static int zfs_uiomove_iov(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio) { - const struct iovec *iov = uio->uio_iov; + struct iovec *iov = uio->uio_iov; size_t skip = uio->uio_skip; ulong_t cnt; @@ -94,7 +94,7 @@ zfs_uiomove_iov(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio) static int zfs_uiomove_bvec_impl(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio) { - const struct bio_vec *bv = uio->uio_bvec; + struct bio_vec *bv = uio->uio_bvec; size_t skip = uio->uio_skip; ulong_t cnt; diff --git a/module/os/linux/zfs/zio_crypt.c b/module/os/linux/zfs/zio_crypt.c deleted file mode 100644 index 28f91ceba1ae..000000000000 --- a/module/os/linux/zfs/zio_crypt.c +++ /dev/null @@ -1,2080 +0,0 @@ -// SPDX-License-Identifier: CDDL-1.0 -/* - * CDDL HEADER START - * - * This file and its contents are supplied under the terms of the - * Common Development and Distribution License ("CDDL"), version 1.0. - * You may only use this file in accordance with the terms of version - * 1.0 of the CDDL. - * - * A full copy of the text of the CDDL should have accompanied this - * source. A copy of the CDDL is also available via the Internet at - * http://www.illumos.org/license/CDDL. - * - * CDDL HEADER END - */ - -/* - * Copyright (c) 2017, Datto, Inc. All rights reserved. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * This file is responsible for handling all of the details of generating - * encryption parameters and performing encryption and authentication. - * - * BLOCK ENCRYPTION PARAMETERS: - * Encryption /Authentication Algorithm Suite (crypt): - * The encryption algorithm, mode, and key length we are going to use. We - * currently support AES in either GCM or CCM modes with 128, 192, and 256 bit - * keys. All authentication is currently done with SHA512-HMAC. - * - * Plaintext: - * The unencrypted data that we want to encrypt. - * - * Initialization Vector (IV): - * An initialization vector for the encryption algorithms. This is used to - * "tweak" the encryption algorithms so that two blocks of the same data are - * encrypted into different ciphertext outputs, thus obfuscating block patterns. - * The supported encryption modes (AES-GCM and AES-CCM) require that an IV is - * never reused with the same encryption key. This value is stored unencrypted - * and must simply be provided to the decryption function. We use a 96 bit IV - * (as recommended by NIST) for all block encryption. For non-dedup blocks we - * derive the IV randomly. The first 64 bits of the IV are stored in the second - * word of DVA[2] and the remaining 32 bits are stored in the upper 32 bits of - * blk_fill. This is safe because encrypted blocks can't use the upper 32 bits - * of blk_fill. We only encrypt level 0 blocks, which normally have a fill count - * of 1. The only exception is for DMU_OT_DNODE objects, where the fill count of - * level 0 blocks is the number of allocated dnodes in that block. The on-disk - * format supports at most 2^15 slots per L0 dnode block, because the maximum - * block size is 16MB (2^24). In either case, for level 0 blocks this number - * will still be smaller than UINT32_MAX so it is safe to store the IV in the - * top 32 bits of blk_fill, while leaving the bottom 32 bits of the fill count - * for the dnode code. - * - * Master key: - * This is the most important secret data of an encrypted dataset. It is used - * along with the salt to generate that actual encryption keys via HKDF. We - * do not use the master key to directly encrypt any data because there are - * theoretical limits on how much data can actually be safely encrypted with - * any encryption mode. The master key is stored encrypted on disk with the - * user's wrapping key. Its length is determined by the encryption algorithm. - * For details on how this is stored see the block comment in dsl_crypt.c - * - * Salt: - * Used as an input to the HKDF function, along with the master key. We use a - * 64 bit salt, stored unencrypted in the first word of DVA[2]. Any given salt - * can be used for encrypting many blocks, so we cache the current salt and the - * associated derived key in zio_crypt_t so we do not need to derive it again - * needlessly. - * - * Encryption Key: - * A secret binary key, generated from an HKDF function used to encrypt and - * decrypt data. - * - * Message Authentication Code (MAC) - * The MAC is an output of authenticated encryption modes such as AES-GCM and - * AES-CCM. Its purpose is to ensure that an attacker cannot modify encrypted - * data on disk and return garbage to the application. Effectively, it is a - * checksum that can not be reproduced by an attacker. We store the MAC in the - * second 128 bits of blk_cksum, leaving the first 128 bits for a truncated - * regular checksum of the ciphertext which can be used for scrubbing. - * - * OBJECT AUTHENTICATION: - * Some object types, such as DMU_OT_MASTER_NODE cannot be encrypted because - * they contain some info that always needs to be readable. To prevent this - * data from being altered, we authenticate this data using SHA512-HMAC. This - * will produce a MAC (similar to the one produced via encryption) which can - * be used to verify the object was not modified. HMACs do not require key - * rotation or IVs, so we can keep up to the full 3 copies of authenticated - * data. - * - * ZIL ENCRYPTION: - * ZIL blocks have their bp written to disk ahead of the associated data, so we - * cannot store the MAC there as we normally do. For these blocks the MAC is - * stored in the embedded checksum within the zil_chain_t header. The salt and - * IV are generated for the block on bp allocation instead of at encryption - * time. In addition, ZIL blocks have some pieces that must be left in plaintext - * for claiming even though all of the sensitive user data still needs to be - * encrypted. The function zio_crypt_init_uios_zil() handles parsing which - * pieces of the block need to be encrypted. All data that is not encrypted is - * authenticated using the AAD mechanisms that the supported encryption modes - * provide for. In order to preserve the semantics of the ZIL for encrypted - * datasets, the ZIL is not protected at the objset level as described below. - * - * DNODE ENCRYPTION: - * Similarly to ZIL blocks, the core part of each dnode_phys_t needs to be left - * in plaintext for scrubbing and claiming, but the bonus buffers might contain - * sensitive user data. The function zio_crypt_init_uios_dnode() handles parsing - * which pieces of the block need to be encrypted. For more details about - * dnode authentication and encryption, see zio_crypt_init_uios_dnode(). - * - * OBJECT SET AUTHENTICATION: - * Up to this point, everything we have encrypted and authenticated has been - * at level 0 (or -2 for the ZIL). If we did not do any further work the - * on-disk format would be susceptible to attacks that deleted or rearranged - * the order of level 0 blocks. Ideally, the cleanest solution would be to - * maintain a tree of authentication MACs going up the bp tree. However, this - * presents a problem for raw sends. Send files do not send information about - * indirect blocks so there would be no convenient way to transfer the MACs and - * they cannot be recalculated on the receive side without the master key which - * would defeat one of the purposes of raw sends in the first place. Instead, - * for the indirect levels of the bp tree, we use a regular SHA512 of the MACs - * from the level below. We also include some portable fields from blk_prop such - * as the lsize and compression algorithm to prevent the data from being - * misinterpreted. - * - * At the objset level, we maintain 2 separate 256 bit MACs in the - * objset_phys_t. The first one is "portable" and is the logical root of the - * MAC tree maintained in the metadnode's bps. The second, is "local" and is - * used as the root MAC for the user accounting objects, which are also not - * transferred via "zfs send". The portable MAC is sent in the DRR_BEGIN payload - * of the send file. The useraccounting code ensures that the useraccounting - * info is not present upon a receive, so the local MAC can simply be cleared - * out at that time. For more info about objset_phys_t authentication, see - * zio_crypt_do_objset_hmacs(). - * - * CONSIDERATIONS FOR DEDUP: - * In order for dedup to work, blocks that we want to dedup with one another - * need to use the same IV and encryption key, so that they will have the same - * ciphertext. Normally, one should never reuse an IV with the same encryption - * key or else AES-GCM and AES-CCM can both actually leak the plaintext of both - * blocks. In this case, however, since we are using the same plaintext as - * well all that we end up with is a duplicate of the original ciphertext we - * already had. As a result, an attacker with read access to the raw disk will - * be able to tell which blocks are the same but this information is given away - * by dedup anyway. In order to get the same IVs and encryption keys for - * equivalent blocks of data we use an HMAC of the plaintext. We use an HMAC - * here so that a reproducible checksum of the plaintext is never available to - * the attacker. The HMAC key is kept alongside the master key, encrypted on - * disk. The first 64 bits of the HMAC are used in place of the random salt, and - * the next 96 bits are used as the IV. As a result of this mechanism, dedup - * will only work within a clone family since encrypted dedup requires use of - * the same master and HMAC keys. - */ - -/* - * After encrypting many blocks with the same key we may start to run up - * against the theoretical limits of how much data can securely be encrypted - * with a single key using the supported encryption modes. The most obvious - * limitation is that our risk of generating 2 equivalent 96 bit IVs increases - * the more IVs we generate (which both GCM and CCM modes strictly forbid). - * This risk actually grows surprisingly quickly over time according to the - * Birthday Problem. With a total IV space of 2^(96 bits), and assuming we have - * generated n IVs with a cryptographically secure RNG, the approximate - * probability p(n) of a collision is given as: - * - * p(n) ~= e^(-n*(n-1)/(2*(2^96))) - * - * [http://www.math.cornell.edu/~mec/2008-2009/TianyiZheng/Birthday.html] - * - * Assuming that we want to ensure that p(n) never goes over 1 / 1 trillion - * we must not write more than 398,065,730 blocks with the same encryption key. - * Therefore, we rotate our keys after 400,000,000 blocks have been written by - * generating a new random 64 bit salt for our HKDF encryption key generation - * function. - */ -#define ZFS_KEY_MAX_SALT_USES_DEFAULT 400000000 -#define ZFS_CURRENT_MAX_SALT_USES \ - (MIN(zfs_key_max_salt_uses, ZFS_KEY_MAX_SALT_USES_DEFAULT)) -static unsigned long zfs_key_max_salt_uses = ZFS_KEY_MAX_SALT_USES_DEFAULT; - -typedef struct blkptr_auth_buf { - uint64_t bab_prop; /* blk_prop - portable mask */ - uint8_t bab_mac[ZIO_DATA_MAC_LEN]; /* MAC from blk_cksum */ - uint64_t bab_pad; /* reserved for future use */ -} blkptr_auth_buf_t; - -const zio_crypt_info_t zio_crypt_table[ZIO_CRYPT_FUNCTIONS] = { - {"", ZC_TYPE_NONE, 0, "inherit"}, - {"", ZC_TYPE_NONE, 0, "on"}, - {"", ZC_TYPE_NONE, 0, "off"}, - {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 16, "aes-128-ccm"}, - {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 24, "aes-192-ccm"}, - {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 32, "aes-256-ccm"}, - {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 16, "aes-128-gcm"}, - {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 24, "aes-192-gcm"}, - {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 32, "aes-256-gcm"} -}; - -void -zio_crypt_key_destroy(zio_crypt_key_t *key) -{ - rw_destroy(&key->zk_salt_lock); - - /* free crypto templates */ - crypto_destroy_ctx_template(key->zk_current_tmpl); - crypto_destroy_ctx_template(key->zk_hmac_tmpl); - - /* zero out sensitive data */ - memset(key, 0, sizeof (zio_crypt_key_t)); -} - -int -zio_crypt_key_init(uint64_t crypt, zio_crypt_key_t *key) -{ - int ret; - crypto_mechanism_t mech = {0}; - uint_t keydata_len; - - ASSERT(key != NULL); - ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); - -/* - * Workaround for GCC 12+ with UBSan enabled deficencies. - * - * GCC 12+ invoked with -fsanitize=undefined incorrectly reports the code - * below as violating -Warray-bounds - */ -#if defined(__GNUC__) && !defined(__clang__) && \ - ((!defined(_KERNEL) && defined(ZFS_UBSAN_ENABLED)) || \ - defined(CONFIG_UBSAN)) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Warray-bounds" -#endif - keydata_len = zio_crypt_table[crypt].ci_keylen; -#if defined(__GNUC__) && !defined(__clang__) && \ - ((!defined(_KERNEL) && defined(ZFS_UBSAN_ENABLED)) || \ - defined(CONFIG_UBSAN)) -#pragma GCC diagnostic pop -#endif - memset(key, 0, sizeof (zio_crypt_key_t)); - rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); - - /* fill keydata buffers and salt with random data */ - ret = random_get_bytes((uint8_t *)&key->zk_guid, sizeof (uint64_t)); - if (ret != 0) - goto error; - - ret = random_get_bytes(key->zk_master_keydata, keydata_len); - if (ret != 0) - goto error; - - ret = random_get_bytes(key->zk_hmac_keydata, SHA512_HMAC_KEYLEN); - if (ret != 0) - goto error; - - ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); - if (ret != 0) - goto error; - - /* derive the current key from the master key */ - ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, - key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, - keydata_len); - if (ret != 0) - goto error; - - /* initialize keys for the ICP */ - key->zk_current_key.ck_data = key->zk_current_keydata; - key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); - - key->zk_hmac_key.ck_data = &key->zk_hmac_key; - key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); - - /* - * Initialize the crypto templates. It's ok if this fails because - * this is just an optimization. - */ - mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname); - ret = crypto_create_ctx_template(&mech, &key->zk_current_key, - &key->zk_current_tmpl); - if (ret != CRYPTO_SUCCESS) - key->zk_current_tmpl = NULL; - - mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); - ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key, - &key->zk_hmac_tmpl); - if (ret != CRYPTO_SUCCESS) - key->zk_hmac_tmpl = NULL; - - key->zk_crypt = crypt; - key->zk_version = ZIO_CRYPT_KEY_CURRENT_VERSION; - key->zk_salt_count = 0; - - return (0); - -error: - zio_crypt_key_destroy(key); - return (ret); -} - -static int -zio_crypt_key_change_salt(zio_crypt_key_t *key) -{ - int ret = 0; - uint8_t salt[ZIO_DATA_SALT_LEN]; - crypto_mechanism_t mech; - uint_t keydata_len = zio_crypt_table[key->zk_crypt].ci_keylen; - - /* generate a new salt */ - ret = random_get_bytes(salt, ZIO_DATA_SALT_LEN); - if (ret != 0) - goto error; - - rw_enter(&key->zk_salt_lock, RW_WRITER); - - /* someone beat us to the salt rotation, just unlock and return */ - if (key->zk_salt_count < ZFS_CURRENT_MAX_SALT_USES) - goto out_unlock; - - /* derive the current key from the master key and the new salt */ - ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, - salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, keydata_len); - if (ret != 0) - goto out_unlock; - - /* assign the salt and reset the usage count */ - memcpy(key->zk_salt, salt, ZIO_DATA_SALT_LEN); - key->zk_salt_count = 0; - - /* destroy the old context template and create the new one */ - crypto_destroy_ctx_template(key->zk_current_tmpl); - ret = crypto_create_ctx_template(&mech, &key->zk_current_key, - &key->zk_current_tmpl); - if (ret != CRYPTO_SUCCESS) - key->zk_current_tmpl = NULL; - - rw_exit(&key->zk_salt_lock); - - return (0); - -out_unlock: - rw_exit(&key->zk_salt_lock); -error: - return (ret); -} - -/* See comment above zfs_key_max_salt_uses definition for details */ -int -zio_crypt_key_get_salt(zio_crypt_key_t *key, uint8_t *salt) -{ - int ret; - boolean_t salt_change; - - rw_enter(&key->zk_salt_lock, RW_READER); - - memcpy(salt, key->zk_salt, ZIO_DATA_SALT_LEN); - salt_change = (atomic_inc_64_nv(&key->zk_salt_count) >= - ZFS_CURRENT_MAX_SALT_USES); - - rw_exit(&key->zk_salt_lock); - - if (salt_change) { - ret = zio_crypt_key_change_salt(key); - if (ret != 0) - goto error; - } - - return (0); - -error: - return (ret); -} - -/* - * This function handles all encryption and decryption in zfs. When - * encrypting it expects puio to reference the plaintext and cuio to - * reference the ciphertext. cuio must have enough space for the - * ciphertext + room for a MAC. datalen should be the length of the - * plaintext / ciphertext alone. - */ -static int -zio_do_crypt_uio(boolean_t encrypt, uint64_t crypt, crypto_key_t *key, - crypto_ctx_template_t tmpl, uint8_t *ivbuf, uint_t datalen, - zfs_uio_t *puio, zfs_uio_t *cuio, uint8_t *authbuf, uint_t auth_len) -{ - int ret; - crypto_data_t plaindata, cipherdata; - CK_AES_CCM_PARAMS ccmp; - CK_AES_GCM_PARAMS gcmp; - crypto_mechanism_t mech; - zio_crypt_info_t crypt_info; - uint_t plain_full_len, maclen; - - ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); - - /* lookup the encryption info */ - crypt_info = zio_crypt_table[crypt]; - - /* the mac will always be the last iovec_t in the cipher uio */ - maclen = cuio->uio_iov[cuio->uio_iovcnt - 1].iov_len; - - ASSERT(maclen <= ZIO_DATA_MAC_LEN); - - /* setup encryption mechanism (same as crypt) */ - mech.cm_type = crypto_mech2id(crypt_info.ci_mechname); - - /* - * Strangely, the ICP requires that plain_full_len must include - * the MAC length when decrypting, even though the UIO does not - * need to have the extra space allocated. - */ - if (encrypt) { - plain_full_len = datalen; - } else { - plain_full_len = datalen + maclen; - } - - /* - * setup encryption params (currently only AES CCM and AES GCM - * are supported) - */ - if (crypt_info.ci_crypt_type == ZC_TYPE_CCM) { - ccmp.ulNonceSize = ZIO_DATA_IV_LEN; - ccmp.ulAuthDataSize = auth_len; - ccmp.authData = authbuf; - ccmp.ulMACSize = maclen; - ccmp.nonce = ivbuf; - ccmp.ulDataSize = plain_full_len; - - mech.cm_param = (char *)(&ccmp); - mech.cm_param_len = sizeof (CK_AES_CCM_PARAMS); - } else { - gcmp.ulIvLen = ZIO_DATA_IV_LEN; - gcmp.ulIvBits = CRYPTO_BYTES2BITS(ZIO_DATA_IV_LEN); - gcmp.ulAADLen = auth_len; - gcmp.pAAD = authbuf; - gcmp.ulTagBits = CRYPTO_BYTES2BITS(maclen); - gcmp.pIv = ivbuf; - - mech.cm_param = (char *)(&gcmp); - mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS); - } - - /* populate the cipher and plain data structs. */ - plaindata.cd_format = CRYPTO_DATA_UIO; - plaindata.cd_offset = 0; - plaindata.cd_uio = puio; - plaindata.cd_length = plain_full_len; - - cipherdata.cd_format = CRYPTO_DATA_UIO; - cipherdata.cd_offset = 0; - cipherdata.cd_uio = cuio; - cipherdata.cd_length = datalen + maclen; - - /* perform the actual encryption */ - if (encrypt) { - ret = crypto_encrypt(&mech, &plaindata, key, tmpl, &cipherdata); - if (ret != CRYPTO_SUCCESS) { - ret = SET_ERROR(EIO); - goto error; - } - } else { - ret = crypto_decrypt(&mech, &cipherdata, key, tmpl, &plaindata); - if (ret != CRYPTO_SUCCESS) { - ASSERT3U(ret, ==, CRYPTO_INVALID_MAC); - ret = SET_ERROR(ECKSUM); - goto error; - } - } - - return (0); - -error: - return (ret); -} - -int -zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv, - uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out) -{ - int ret; - zfs_uio_t puio, cuio; - uint64_t aad[3]; - iovec_t plain_iovecs[2], cipher_iovecs[3]; - uint64_t crypt = key->zk_crypt; - uint_t enc_len, keydata_len, aad_len; - - ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); - - keydata_len = zio_crypt_table[crypt].ci_keylen; - - /* generate iv for wrapping the master and hmac key */ - ret = random_get_pseudo_bytes(iv, WRAPPING_IV_LEN); - if (ret != 0) - goto error; - - /* initialize zfs_uio_ts */ - plain_iovecs[0].iov_base = key->zk_master_keydata; - plain_iovecs[0].iov_len = keydata_len; - plain_iovecs[1].iov_base = key->zk_hmac_keydata; - plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; - - cipher_iovecs[0].iov_base = keydata_out; - cipher_iovecs[0].iov_len = keydata_len; - cipher_iovecs[1].iov_base = hmac_keydata_out; - cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; - cipher_iovecs[2].iov_base = mac; - cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN; - - /* - * Although we don't support writing to the old format, we do - * support rewrapping the key so that the user can move and - * quarantine datasets on the old format. - */ - if (key->zk_version == 0) { - aad_len = sizeof (uint64_t); - aad[0] = LE_64(key->zk_guid); - } else { - ASSERT3U(key->zk_version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); - aad_len = sizeof (uint64_t) * 3; - aad[0] = LE_64(key->zk_guid); - aad[1] = LE_64(crypt); - aad[2] = LE_64(key->zk_version); - } - - enc_len = zio_crypt_table[crypt].ci_keylen + SHA512_HMAC_KEYLEN; - puio.uio_iov = plain_iovecs; - puio.uio_iovcnt = 2; - puio.uio_segflg = UIO_SYSSPACE; - cuio.uio_iov = cipher_iovecs; - cuio.uio_iovcnt = 3; - cuio.uio_segflg = UIO_SYSSPACE; - - /* encrypt the keys and store the resulting ciphertext and mac */ - ret = zio_do_crypt_uio(B_TRUE, crypt, cwkey, NULL, iv, enc_len, - &puio, &cuio, (uint8_t *)aad, aad_len); - if (ret != 0) - goto error; - - return (0); - -error: - return (ret); -} - -int -zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version, - uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata, uint8_t *iv, - uint8_t *mac, zio_crypt_key_t *key) -{ - crypto_mechanism_t mech; - zfs_uio_t puio, cuio; - uint64_t aad[3]; - iovec_t plain_iovecs[2], cipher_iovecs[3]; - uint_t enc_len, keydata_len, aad_len; - int ret; - - ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); - - rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); - - keydata_len = zio_crypt_table[crypt].ci_keylen; - - /* initialize zfs_uio_ts */ - plain_iovecs[0].iov_base = key->zk_master_keydata; - plain_iovecs[0].iov_len = keydata_len; - plain_iovecs[1].iov_base = key->zk_hmac_keydata; - plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; - - cipher_iovecs[0].iov_base = keydata; - cipher_iovecs[0].iov_len = keydata_len; - cipher_iovecs[1].iov_base = hmac_keydata; - cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; - cipher_iovecs[2].iov_base = mac; - cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN; - - if (version == 0) { - aad_len = sizeof (uint64_t); - aad[0] = LE_64(guid); - } else { - ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); - aad_len = sizeof (uint64_t) * 3; - aad[0] = LE_64(guid); - aad[1] = LE_64(crypt); - aad[2] = LE_64(version); - } - - enc_len = keydata_len + SHA512_HMAC_KEYLEN; - puio.uio_iov = plain_iovecs; - puio.uio_segflg = UIO_SYSSPACE; - puio.uio_iovcnt = 2; - cuio.uio_iov = cipher_iovecs; - cuio.uio_iovcnt = 3; - cuio.uio_segflg = UIO_SYSSPACE; - - /* decrypt the keys and store the result in the output buffers */ - ret = zio_do_crypt_uio(B_FALSE, crypt, cwkey, NULL, iv, enc_len, - &puio, &cuio, (uint8_t *)aad, aad_len); - if (ret != 0) - goto error; - - /* generate a fresh salt */ - ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); - if (ret != 0) - goto error; - - /* derive the current key from the master key */ - ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, - key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, - keydata_len); - if (ret != 0) - goto error; - - /* initialize keys for ICP */ - key->zk_current_key.ck_data = key->zk_current_keydata; - key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); - - key->zk_hmac_key.ck_data = key->zk_hmac_keydata; - key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); - - /* - * Initialize the crypto templates. It's ok if this fails because - * this is just an optimization. - */ - mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname); - ret = crypto_create_ctx_template(&mech, &key->zk_current_key, - &key->zk_current_tmpl); - if (ret != CRYPTO_SUCCESS) - key->zk_current_tmpl = NULL; - - mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); - ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key, - &key->zk_hmac_tmpl); - if (ret != CRYPTO_SUCCESS) - key->zk_hmac_tmpl = NULL; - - key->zk_crypt = crypt; - key->zk_version = version; - key->zk_guid = guid; - key->zk_salt_count = 0; - - return (0); - -error: - zio_crypt_key_destroy(key); - return (ret); -} - -int -zio_crypt_generate_iv(uint8_t *ivbuf) -{ - int ret; - - /* randomly generate the IV */ - ret = random_get_pseudo_bytes(ivbuf, ZIO_DATA_IV_LEN); - if (ret != 0) - goto error; - - return (0); - -error: - memset(ivbuf, 0, ZIO_DATA_IV_LEN); - return (ret); -} - -int -zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen, - uint8_t *digestbuf, uint_t digestlen) -{ - int ret; - crypto_mechanism_t mech; - crypto_data_t in_data, digest_data; - uint8_t raw_digestbuf[SHA512_DIGEST_LENGTH]; - - ASSERT3U(digestlen, <=, SHA512_DIGEST_LENGTH); - - /* initialize sha512-hmac mechanism and crypto data */ - mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); - mech.cm_param = NULL; - mech.cm_param_len = 0; - - /* initialize the crypto data */ - in_data.cd_format = CRYPTO_DATA_RAW; - in_data.cd_offset = 0; - in_data.cd_length = datalen; - in_data.cd_raw.iov_base = (char *)data; - in_data.cd_raw.iov_len = in_data.cd_length; - - digest_data.cd_format = CRYPTO_DATA_RAW; - digest_data.cd_offset = 0; - digest_data.cd_length = SHA512_DIGEST_LENGTH; - digest_data.cd_raw.iov_base = (char *)raw_digestbuf; - digest_data.cd_raw.iov_len = digest_data.cd_length; - - /* generate the hmac */ - ret = crypto_mac(&mech, &in_data, &key->zk_hmac_key, key->zk_hmac_tmpl, - &digest_data); - if (ret != CRYPTO_SUCCESS) { - ret = SET_ERROR(EIO); - goto error; - } - - memcpy(digestbuf, raw_digestbuf, digestlen); - - return (0); - -error: - memset(digestbuf, 0, digestlen); - return (ret); -} - -int -zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t *key, uint8_t *data, - uint_t datalen, uint8_t *ivbuf, uint8_t *salt) -{ - int ret; - uint8_t digestbuf[SHA512_DIGEST_LENGTH]; - - ret = zio_crypt_do_hmac(key, data, datalen, - digestbuf, SHA512_DIGEST_LENGTH); - if (ret != 0) - return (ret); - - memcpy(salt, digestbuf, ZIO_DATA_SALT_LEN); - memcpy(ivbuf, digestbuf + ZIO_DATA_SALT_LEN, ZIO_DATA_IV_LEN); - - return (0); -} - -/* - * The following functions are used to encode and decode encryption parameters - * into blkptr_t and zil_header_t. The ICP wants to use these parameters as - * byte strings, which normally means that these strings would not need to deal - * with byteswapping at all. However, both blkptr_t and zil_header_t may be - * byteswapped by lower layers and so we must "undo" that byteswap here upon - * decoding and encoding in a non-native byteorder. These functions require - * that the byteorder bit is correct before being called. - */ -void -zio_crypt_encode_params_bp(blkptr_t *bp, uint8_t *salt, uint8_t *iv) -{ - uint64_t val64; - uint32_t val32; - - ASSERT(BP_IS_ENCRYPTED(bp)); - - if (!BP_SHOULD_BYTESWAP(bp)) { - memcpy(&bp->blk_dva[2].dva_word[0], salt, sizeof (uint64_t)); - memcpy(&bp->blk_dva[2].dva_word[1], iv, sizeof (uint64_t)); - memcpy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t)); - BP_SET_IV2(bp, val32); - } else { - memcpy(&val64, salt, sizeof (uint64_t)); - bp->blk_dva[2].dva_word[0] = BSWAP_64(val64); - - memcpy(&val64, iv, sizeof (uint64_t)); - bp->blk_dva[2].dva_word[1] = BSWAP_64(val64); - - memcpy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t)); - BP_SET_IV2(bp, BSWAP_32(val32)); - } -} - -void -zio_crypt_decode_params_bp(const blkptr_t *bp, uint8_t *salt, uint8_t *iv) -{ - uint64_t val64; - uint32_t val32; - - ASSERT(BP_IS_PROTECTED(bp)); - - /* for convenience, so callers don't need to check */ - if (BP_IS_AUTHENTICATED(bp)) { - memset(salt, 0, ZIO_DATA_SALT_LEN); - memset(iv, 0, ZIO_DATA_IV_LEN); - return; - } - - if (!BP_SHOULD_BYTESWAP(bp)) { - memcpy(salt, &bp->blk_dva[2].dva_word[0], sizeof (uint64_t)); - memcpy(iv, &bp->blk_dva[2].dva_word[1], sizeof (uint64_t)); - - val32 = (uint32_t)BP_GET_IV2(bp); - memcpy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t)); - } else { - val64 = BSWAP_64(bp->blk_dva[2].dva_word[0]); - memcpy(salt, &val64, sizeof (uint64_t)); - - val64 = BSWAP_64(bp->blk_dva[2].dva_word[1]); - memcpy(iv, &val64, sizeof (uint64_t)); - - val32 = BSWAP_32((uint32_t)BP_GET_IV2(bp)); - memcpy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t)); - } -} - -void -zio_crypt_encode_mac_bp(blkptr_t *bp, uint8_t *mac) -{ - uint64_t val64; - - ASSERT(BP_USES_CRYPT(bp)); - ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_OBJSET); - - if (!BP_SHOULD_BYTESWAP(bp)) { - memcpy(&bp->blk_cksum.zc_word[2], mac, sizeof (uint64_t)); - memcpy(&bp->blk_cksum.zc_word[3], mac + sizeof (uint64_t), - sizeof (uint64_t)); - } else { - memcpy(&val64, mac, sizeof (uint64_t)); - bp->blk_cksum.zc_word[2] = BSWAP_64(val64); - - memcpy(&val64, mac + sizeof (uint64_t), sizeof (uint64_t)); - bp->blk_cksum.zc_word[3] = BSWAP_64(val64); - } -} - -void -zio_crypt_decode_mac_bp(const blkptr_t *bp, uint8_t *mac) -{ - uint64_t val64; - - ASSERT(BP_USES_CRYPT(bp) || BP_IS_HOLE(bp)); - - /* for convenience, so callers don't need to check */ - if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) { - memset(mac, 0, ZIO_DATA_MAC_LEN); - return; - } - - if (!BP_SHOULD_BYTESWAP(bp)) { - memcpy(mac, &bp->blk_cksum.zc_word[2], sizeof (uint64_t)); - memcpy(mac + sizeof (uint64_t), &bp->blk_cksum.zc_word[3], - sizeof (uint64_t)); - } else { - val64 = BSWAP_64(bp->blk_cksum.zc_word[2]); - memcpy(mac, &val64, sizeof (uint64_t)); - - val64 = BSWAP_64(bp->blk_cksum.zc_word[3]); - memcpy(mac + sizeof (uint64_t), &val64, sizeof (uint64_t)); - } -} - -void -zio_crypt_encode_mac_zil(void *data, uint8_t *mac) -{ - zil_chain_t *zilc = data; - - memcpy(&zilc->zc_eck.zec_cksum.zc_word[2], mac, sizeof (uint64_t)); - memcpy(&zilc->zc_eck.zec_cksum.zc_word[3], mac + sizeof (uint64_t), - sizeof (uint64_t)); -} - -void -zio_crypt_decode_mac_zil(const void *data, uint8_t *mac) -{ - /* - * The ZIL MAC is embedded in the block it protects, which will - * not have been byteswapped by the time this function has been called. - * As a result, we don't need to worry about byteswapping the MAC. - */ - const zil_chain_t *zilc = data; - - memcpy(mac, &zilc->zc_eck.zec_cksum.zc_word[2], sizeof (uint64_t)); - memcpy(mac + sizeof (uint64_t), &zilc->zc_eck.zec_cksum.zc_word[3], - sizeof (uint64_t)); -} - -/* - * This routine takes a block of dnodes (src_abd) and copies only the bonus - * buffers to the same offsets in the dst buffer. datalen should be the size - * of both the src_abd and the dst buffer (not just the length of the bonus - * buffers). - */ -void -zio_crypt_copy_dnode_bonus(abd_t *src_abd, uint8_t *dst, uint_t datalen) -{ - uint_t i, max_dnp = datalen >> DNODE_SHIFT; - uint8_t *src; - dnode_phys_t *dnp, *sdnp, *ddnp; - - src = abd_borrow_buf_copy(src_abd, datalen); - - sdnp = (dnode_phys_t *)src; - ddnp = (dnode_phys_t *)dst; - - for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { - dnp = &sdnp[i]; - if (dnp->dn_type != DMU_OT_NONE && - DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) && - dnp->dn_bonuslen != 0) { - memcpy(DN_BONUS(&ddnp[i]), DN_BONUS(dnp), - DN_MAX_BONUS_LEN(dnp)); - } - } - - abd_return_buf(src_abd, src, datalen); -} - -/* - * This function decides what fields from blk_prop are included in - * the on-disk various MAC algorithms. - */ -static void -zio_crypt_bp_zero_nonportable_blkprop(blkptr_t *bp, uint64_t version) -{ - /* - * Version 0 did not properly zero out all non-portable fields - * as it should have done. We maintain this code so that we can - * do read-only imports of pools on this version. - */ - if (version == 0) { - BP_SET_DEDUP(bp, 0); - BP_SET_CHECKSUM(bp, 0); - BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE); - return; - } - - ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); - - /* - * The hole_birth feature might set these fields even if this bp - * is a hole. We zero them out here to guarantee that raw sends - * will function with or without the feature. - */ - if (BP_IS_HOLE(bp)) { - bp->blk_prop = 0ULL; - return; - } - - /* - * At L0 we want to verify these fields to ensure that data blocks - * can not be reinterpreted. For instance, we do not want an attacker - * to trick us into returning raw lz4 compressed data to the user - * by modifying the compression bits. At higher levels, we cannot - * enforce this policy since raw sends do not convey any information - * about indirect blocks, so these values might be different on the - * receive side. Fortunately, this does not open any new attack - * vectors, since any alterations that can be made to a higher level - * bp must still verify the correct order of the layer below it. - */ - if (BP_GET_LEVEL(bp) != 0) { - BP_SET_BYTEORDER(bp, 0); - BP_SET_COMPRESS(bp, 0); - - /* - * psize cannot be set to zero or it will trigger - * asserts, but the value doesn't really matter as - * long as it is constant. - */ - BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE); - } - - BP_SET_DEDUP(bp, 0); - BP_SET_CHECKSUM(bp, 0); -} - -static void -zio_crypt_bp_auth_init(uint64_t version, boolean_t should_bswap, blkptr_t *bp, - blkptr_auth_buf_t *bab, uint_t *bab_len) -{ - blkptr_t tmpbp = *bp; - - if (should_bswap) - byteswap_uint64_array(&tmpbp, sizeof (blkptr_t)); - - ASSERT(BP_USES_CRYPT(&tmpbp) || BP_IS_HOLE(&tmpbp)); - ASSERT0(BP_IS_EMBEDDED(&tmpbp)); - - zio_crypt_decode_mac_bp(&tmpbp, bab->bab_mac); - - /* - * We always MAC blk_prop in LE to ensure portability. This - * must be done after decoding the mac, since the endianness - * will get zero'd out here. - */ - zio_crypt_bp_zero_nonportable_blkprop(&tmpbp, version); - bab->bab_prop = LE_64(tmpbp.blk_prop); - bab->bab_pad = 0ULL; - - /* version 0 did not include the padding */ - *bab_len = sizeof (blkptr_auth_buf_t); - if (version == 0) - *bab_len -= sizeof (uint64_t); -} - -static int -zio_crypt_bp_do_hmac_updates(crypto_context_t ctx, uint64_t version, - boolean_t should_bswap, blkptr_t *bp) -{ - int ret; - uint_t bab_len; - blkptr_auth_buf_t bab; - crypto_data_t cd; - - zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); - cd.cd_format = CRYPTO_DATA_RAW; - cd.cd_offset = 0; - cd.cd_length = bab_len; - cd.cd_raw.iov_base = (char *)&bab; - cd.cd_raw.iov_len = cd.cd_length; - - ret = crypto_mac_update(ctx, &cd); - if (ret != CRYPTO_SUCCESS) { - ret = SET_ERROR(EIO); - goto error; - } - - return (0); - -error: - return (ret); -} - -static void -zio_crypt_bp_do_indrect_checksum_updates(SHA2_CTX *ctx, uint64_t version, - boolean_t should_bswap, blkptr_t *bp) -{ - uint_t bab_len; - blkptr_auth_buf_t bab; - - zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); - SHA2Update(ctx, &bab, bab_len); -} - -static void -zio_crypt_bp_do_aad_updates(uint8_t **aadp, uint_t *aad_len, uint64_t version, - boolean_t should_bswap, blkptr_t *bp) -{ - uint_t bab_len; - blkptr_auth_buf_t bab; - - zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); - memcpy(*aadp, &bab, bab_len); - *aadp += bab_len; - *aad_len += bab_len; -} - -static int -zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx, uint64_t version, - boolean_t should_bswap, dnode_phys_t *dnp) -{ - int ret, i; - dnode_phys_t *adnp, tmp_dncore; - size_t dn_core_size = offsetof(dnode_phys_t, dn_blkptr); - boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); - crypto_data_t cd; - - cd.cd_format = CRYPTO_DATA_RAW; - cd.cd_offset = 0; - - /* - * Authenticate the core dnode (masking out non-portable bits). - * We only copy the first 64 bytes we operate on to avoid the overhead - * of copying 512-64 unneeded bytes. The compiler seems to be fine - * with that. - */ - memcpy(&tmp_dncore, dnp, dn_core_size); - adnp = &tmp_dncore; - - if (le_bswap) { - adnp->dn_datablkszsec = BSWAP_16(adnp->dn_datablkszsec); - adnp->dn_bonuslen = BSWAP_16(adnp->dn_bonuslen); - adnp->dn_maxblkid = BSWAP_64(adnp->dn_maxblkid); - adnp->dn_used = BSWAP_64(adnp->dn_used); - } - adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK; - adnp->dn_used = 0; - - cd.cd_length = dn_core_size; - cd.cd_raw.iov_base = (char *)adnp; - cd.cd_raw.iov_len = cd.cd_length; - - ret = crypto_mac_update(ctx, &cd); - if (ret != CRYPTO_SUCCESS) { - ret = SET_ERROR(EIO); - goto error; - } - - for (i = 0; i < dnp->dn_nblkptr; i++) { - ret = zio_crypt_bp_do_hmac_updates(ctx, version, - should_bswap, &dnp->dn_blkptr[i]); - if (ret != 0) - goto error; - } - - if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { - ret = zio_crypt_bp_do_hmac_updates(ctx, version, - should_bswap, DN_SPILL_BLKPTR(dnp)); - if (ret != 0) - goto error; - } - - return (0); - -error: - return (ret); -} - -/* - * objset_phys_t blocks introduce a number of exceptions to the normal - * authentication process. objset_phys_t's contain 2 separate HMACS for - * protecting the integrity of their data. The portable_mac protects the - * metadnode. This MAC can be sent with a raw send and protects against - * reordering of data within the metadnode. The local_mac protects the user - * accounting objects which are not sent from one system to another. - * - * In addition, objset blocks are the only blocks that can be modified and - * written to disk without the key loaded under certain circumstances. During - * zil_claim() we need to be able to update the zil_header_t to complete - * claiming log blocks and during raw receives we need to write out the - * portable_mac from the send file. Both of these actions are possible - * because these fields are not protected by either MAC so neither one will - * need to modify the MACs without the key. However, when the modified blocks - * are written out they will be byteswapped into the host machine's native - * endianness which will modify fields protected by the MAC. As a result, MAC - * calculation for objset blocks works slightly differently from other block - * types. Where other block types MAC the data in whatever endianness is - * written to disk, objset blocks always MAC little endian version of their - * values. In the code, should_bswap is the value from BP_SHOULD_BYTESWAP() - * and le_bswap indicates whether a byteswap is needed to get this block - * into little endian format. - */ -int -zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen, - boolean_t should_bswap, uint8_t *portable_mac, uint8_t *local_mac) -{ - int ret; - crypto_mechanism_t mech; - crypto_context_t ctx; - crypto_data_t cd; - objset_phys_t *osp = data; - uint64_t intval; - boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); - uint8_t raw_portable_mac[SHA512_DIGEST_LENGTH]; - uint8_t raw_local_mac[SHA512_DIGEST_LENGTH]; - - /* initialize HMAC mechanism */ - mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); - mech.cm_param = NULL; - mech.cm_param_len = 0; - - cd.cd_format = CRYPTO_DATA_RAW; - cd.cd_offset = 0; - - /* calculate the portable MAC from the portable fields and metadnode */ - ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx); - if (ret != CRYPTO_SUCCESS) { - ret = SET_ERROR(EIO); - goto error; - } - - /* add in the os_type */ - intval = (le_bswap) ? osp->os_type : BSWAP_64(osp->os_type); - cd.cd_length = sizeof (uint64_t); - cd.cd_raw.iov_base = (char *)&intval; - cd.cd_raw.iov_len = cd.cd_length; - - ret = crypto_mac_update(ctx, &cd); - if (ret != CRYPTO_SUCCESS) { - ret = SET_ERROR(EIO); - goto error; - } - - /* add in the portable os_flags */ - intval = osp->os_flags; - if (should_bswap) - intval = BSWAP_64(intval); - intval &= OBJSET_CRYPT_PORTABLE_FLAGS_MASK; - if (!ZFS_HOST_BYTEORDER) - intval = BSWAP_64(intval); - - cd.cd_length = sizeof (uint64_t); - cd.cd_raw.iov_base = (char *)&intval; - cd.cd_raw.iov_len = cd.cd_length; - - ret = crypto_mac_update(ctx, &cd); - if (ret != CRYPTO_SUCCESS) { - ret = SET_ERROR(EIO); - goto error; - } - - /* add in fields from the metadnode */ - ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, - should_bswap, &osp->os_meta_dnode); - if (ret) - goto error; - - /* store the final digest in a temporary buffer and copy what we need */ - cd.cd_length = SHA512_DIGEST_LENGTH; - cd.cd_raw.iov_base = (char *)raw_portable_mac; - cd.cd_raw.iov_len = cd.cd_length; - - ret = crypto_mac_final(ctx, &cd); - if (ret != CRYPTO_SUCCESS) { - ret = SET_ERROR(EIO); - goto error; - } - - memcpy(portable_mac, raw_portable_mac, ZIO_OBJSET_MAC_LEN); - - /* - * This is necessary here as we check next whether - * OBJSET_FLAG_USERACCOUNTING_COMPLETE is set in order to - * decide if the local_mac should be zeroed out. That flag will always - * be set by dmu_objset_id_quota_upgrade_cb() and - * dmu_objset_userspace_upgrade_cb() if useraccounting has been - * completed. - */ - intval = osp->os_flags; - if (should_bswap) - intval = BSWAP_64(intval); - boolean_t uacct_incomplete = - !(intval & OBJSET_FLAG_USERACCOUNTING_COMPLETE); - - /* - * The local MAC protects the user, group and project accounting. - * If these objects are not present, the local MAC is zeroed out. - */ - if (uacct_incomplete || - (datalen >= OBJSET_PHYS_SIZE_V3 && - osp->os_userused_dnode.dn_type == DMU_OT_NONE && - osp->os_groupused_dnode.dn_type == DMU_OT_NONE && - osp->os_projectused_dnode.dn_type == DMU_OT_NONE) || - (datalen >= OBJSET_PHYS_SIZE_V2 && - osp->os_userused_dnode.dn_type == DMU_OT_NONE && - osp->os_groupused_dnode.dn_type == DMU_OT_NONE) || - (datalen <= OBJSET_PHYS_SIZE_V1)) { - memset(local_mac, 0, ZIO_OBJSET_MAC_LEN); - return (0); - } - - /* calculate the local MAC from the userused and groupused dnodes */ - ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx); - if (ret != CRYPTO_SUCCESS) { - ret = SET_ERROR(EIO); - goto error; - } - - /* add in the non-portable os_flags */ - intval = osp->os_flags; - if (should_bswap) - intval = BSWAP_64(intval); - intval &= ~OBJSET_CRYPT_PORTABLE_FLAGS_MASK; - if (!ZFS_HOST_BYTEORDER) - intval = BSWAP_64(intval); - - cd.cd_length = sizeof (uint64_t); - cd.cd_raw.iov_base = (char *)&intval; - cd.cd_raw.iov_len = cd.cd_length; - - ret = crypto_mac_update(ctx, &cd); - if (ret != CRYPTO_SUCCESS) { - ret = SET_ERROR(EIO); - goto error; - } - - /* add in fields from the user accounting dnodes */ - if (osp->os_userused_dnode.dn_type != DMU_OT_NONE) { - ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, - should_bswap, &osp->os_userused_dnode); - if (ret) - goto error; - } - - if (osp->os_groupused_dnode.dn_type != DMU_OT_NONE) { - ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, - should_bswap, &osp->os_groupused_dnode); - if (ret) - goto error; - } - - if (osp->os_projectused_dnode.dn_type != DMU_OT_NONE && - datalen >= OBJSET_PHYS_SIZE_V3) { - ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, - should_bswap, &osp->os_projectused_dnode); - if (ret) - goto error; - } - - /* store the final digest in a temporary buffer and copy what we need */ - cd.cd_length = SHA512_DIGEST_LENGTH; - cd.cd_raw.iov_base = (char *)raw_local_mac; - cd.cd_raw.iov_len = cd.cd_length; - - ret = crypto_mac_final(ctx, &cd); - if (ret != CRYPTO_SUCCESS) { - ret = SET_ERROR(EIO); - goto error; - } - - memcpy(local_mac, raw_local_mac, ZIO_OBJSET_MAC_LEN); - - return (0); - -error: - memset(portable_mac, 0, ZIO_OBJSET_MAC_LEN); - memset(local_mac, 0, ZIO_OBJSET_MAC_LEN); - return (ret); -} - -static void -zio_crypt_destroy_uio(zfs_uio_t *uio) -{ - if (uio->uio_iov) - kmem_free(uio->uio_iov, uio->uio_iovcnt * sizeof (iovec_t)); -} - -/* - * This function parses an uncompressed indirect block and returns a checksum - * of all the portable fields from all of the contained bps. The portable - * fields are the MAC and all of the fields from blk_prop except for the dedup, - * checksum, and psize bits. For an explanation of the purpose of this, see - * the comment block on object set authentication. - */ -static int -zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate, void *buf, - uint_t datalen, uint64_t version, boolean_t byteswap, uint8_t *cksum) -{ - blkptr_t *bp; - int i, epb = datalen >> SPA_BLKPTRSHIFT; - SHA2_CTX ctx; - uint8_t digestbuf[SHA512_DIGEST_LENGTH]; - - /* checksum all of the MACs from the layer below */ - SHA2Init(SHA512, &ctx); - for (i = 0, bp = buf; i < epb; i++, bp++) { - zio_crypt_bp_do_indrect_checksum_updates(&ctx, version, - byteswap, bp); - } - SHA2Final(digestbuf, &ctx); - - if (generate) { - memcpy(cksum, digestbuf, ZIO_DATA_MAC_LEN); - return (0); - } - - if (memcmp(digestbuf, cksum, ZIO_DATA_MAC_LEN) != 0) - return (SET_ERROR(ECKSUM)); - - return (0); -} - -int -zio_crypt_do_indirect_mac_checksum(boolean_t generate, void *buf, - uint_t datalen, boolean_t byteswap, uint8_t *cksum) -{ - int ret; - - /* - * Unfortunately, callers of this function will not always have - * easy access to the on-disk format version. This info is - * normally found in the DSL Crypto Key, but the checksum-of-MACs - * is expected to be verifiable even when the key isn't loaded. - * Here, instead of doing a ZAP lookup for the version for each - * zio, we simply try both existing formats. - */ - ret = zio_crypt_do_indirect_mac_checksum_impl(generate, buf, - datalen, ZIO_CRYPT_KEY_CURRENT_VERSION, byteswap, cksum); - if (ret == ECKSUM) { - ASSERT(!generate); - ret = zio_crypt_do_indirect_mac_checksum_impl(generate, - buf, datalen, 0, byteswap, cksum); - } - - return (ret); -} - -int -zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate, abd_t *abd, - uint_t datalen, boolean_t byteswap, uint8_t *cksum) -{ - int ret; - void *buf; - - buf = abd_borrow_buf_copy(abd, datalen); - ret = zio_crypt_do_indirect_mac_checksum(generate, buf, datalen, - byteswap, cksum); - abd_return_buf(abd, buf, datalen); - - return (ret); -} - -/* - * Special case handling routine for encrypting / decrypting ZIL blocks. - * We do not check for the older ZIL chain because the encryption feature - * was not available before the newer ZIL chain was introduced. The goal - * here is to encrypt everything except the blkptr_t of a lr_write_t and - * the zil_chain_t header. Everything that is not encrypted is authenticated. - */ -static int -zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf, - uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, zfs_uio_t *puio, - zfs_uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len, - boolean_t *no_crypt) -{ - int ret; - uint64_t txtype, lr_len, nused; - uint_t nr_src, nr_dst, crypt_len; - uint_t aad_len = 0, nr_iovecs = 0, total_len = 0; - iovec_t *src_iovecs = NULL, *dst_iovecs = NULL; - uint8_t *src, *dst, *slrp, *dlrp, *blkend, *aadp; - zil_chain_t *zilc; - lr_t *lr; - uint8_t *aadbuf = zio_buf_alloc(datalen); - - /* cipherbuf always needs an extra iovec for the MAC */ - if (encrypt) { - src = plainbuf; - dst = cipherbuf; - nr_src = 0; - nr_dst = 1; - } else { - src = cipherbuf; - dst = plainbuf; - nr_src = 1; - nr_dst = 0; - } - memset(dst, 0, datalen); - - /* find the start and end record of the log block */ - zilc = (zil_chain_t *)src; - slrp = src + sizeof (zil_chain_t); - aadp = aadbuf; - nused = ((byteswap) ? BSWAP_64(zilc->zc_nused) : zilc->zc_nused); - ASSERT3U(nused, >=, sizeof (zil_chain_t)); - ASSERT3U(nused, <=, datalen); - blkend = src + nused; - - /* calculate the number of encrypted iovecs we will need */ - for (; slrp < blkend; slrp += lr_len) { - lr = (lr_t *)slrp; - - if (!byteswap) { - txtype = lr->lrc_txtype; - lr_len = lr->lrc_reclen; - } else { - txtype = BSWAP_64(lr->lrc_txtype); - lr_len = BSWAP_64(lr->lrc_reclen); - } - ASSERT3U(lr_len, >=, sizeof (lr_t)); - ASSERT3U(lr_len, <=, blkend - slrp); - - nr_iovecs++; - if (txtype == TX_WRITE && lr_len != sizeof (lr_write_t)) - nr_iovecs++; - } - - nr_src += nr_iovecs; - nr_dst += nr_iovecs; - - /* allocate the iovec arrays */ - if (nr_src != 0) { - src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP); - if (src_iovecs == NULL) { - ret = SET_ERROR(ENOMEM); - goto error; - } - } - - if (nr_dst != 0) { - dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP); - if (dst_iovecs == NULL) { - ret = SET_ERROR(ENOMEM); - goto error; - } - } - - /* - * Copy the plain zil header over and authenticate everything except - * the checksum that will store our MAC. If we are writing the data - * the embedded checksum will not have been calculated yet, so we don't - * authenticate that. - */ - memcpy(dst, src, sizeof (zil_chain_t)); - memcpy(aadp, src, sizeof (zil_chain_t) - sizeof (zio_eck_t)); - aadp += sizeof (zil_chain_t) - sizeof (zio_eck_t); - aad_len += sizeof (zil_chain_t) - sizeof (zio_eck_t); - - /* loop over records again, filling in iovecs */ - nr_iovecs = 0; - slrp = src + sizeof (zil_chain_t); - dlrp = dst + sizeof (zil_chain_t); - - for (; slrp < blkend; slrp += lr_len, dlrp += lr_len) { - lr = (lr_t *)slrp; - - if (!byteswap) { - txtype = lr->lrc_txtype; - lr_len = lr->lrc_reclen; - } else { - txtype = BSWAP_64(lr->lrc_txtype); - lr_len = BSWAP_64(lr->lrc_reclen); - } - - /* copy the common lr_t */ - memcpy(dlrp, slrp, sizeof (lr_t)); - memcpy(aadp, slrp, sizeof (lr_t)); - aadp += sizeof (lr_t); - aad_len += sizeof (lr_t); - - ASSERT3P(src_iovecs, !=, NULL); - ASSERT3P(dst_iovecs, !=, NULL); - - /* - * If this is a TX_WRITE record we want to encrypt everything - * except the bp if exists. If the bp does exist we want to - * authenticate it. - */ - if (txtype == TX_WRITE) { - const size_t o = offsetof(lr_write_t, lr_blkptr); - crypt_len = o - sizeof (lr_t); - src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_t); - src_iovecs[nr_iovecs].iov_len = crypt_len; - dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_t); - dst_iovecs[nr_iovecs].iov_len = crypt_len; - - /* copy the bp now since it will not be encrypted */ - memcpy(dlrp + o, slrp + o, sizeof (blkptr_t)); - memcpy(aadp, slrp + o, sizeof (blkptr_t)); - aadp += sizeof (blkptr_t); - aad_len += sizeof (blkptr_t); - nr_iovecs++; - total_len += crypt_len; - - if (lr_len != sizeof (lr_write_t)) { - crypt_len = lr_len - sizeof (lr_write_t); - src_iovecs[nr_iovecs].iov_base = - slrp + sizeof (lr_write_t); - src_iovecs[nr_iovecs].iov_len = crypt_len; - dst_iovecs[nr_iovecs].iov_base = - dlrp + sizeof (lr_write_t); - dst_iovecs[nr_iovecs].iov_len = crypt_len; - nr_iovecs++; - total_len += crypt_len; - } - } else if (txtype == TX_CLONE_RANGE) { - const size_t o = offsetof(lr_clone_range_t, lr_nbps); - crypt_len = o - sizeof (lr_t); - src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_t); - src_iovecs[nr_iovecs].iov_len = crypt_len; - dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_t); - dst_iovecs[nr_iovecs].iov_len = crypt_len; - - /* copy the bps now since they will not be encrypted */ - memcpy(dlrp + o, slrp + o, lr_len - o); - memcpy(aadp, slrp + o, lr_len - o); - aadp += lr_len - o; - aad_len += lr_len - o; - nr_iovecs++; - total_len += crypt_len; - } else { - crypt_len = lr_len - sizeof (lr_t); - src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_t); - src_iovecs[nr_iovecs].iov_len = crypt_len; - dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_t); - dst_iovecs[nr_iovecs].iov_len = crypt_len; - nr_iovecs++; - total_len += crypt_len; - } - } - - *no_crypt = (nr_iovecs == 0); - *enc_len = total_len; - *authbuf = aadbuf; - *auth_len = aad_len; - - if (encrypt) { - puio->uio_iov = src_iovecs; - puio->uio_iovcnt = nr_src; - cuio->uio_iov = dst_iovecs; - cuio->uio_iovcnt = nr_dst; - } else { - puio->uio_iov = dst_iovecs; - puio->uio_iovcnt = nr_dst; - cuio->uio_iov = src_iovecs; - cuio->uio_iovcnt = nr_src; - } - - return (0); - -error: - zio_buf_free(aadbuf, datalen); - if (src_iovecs != NULL) - kmem_free(src_iovecs, nr_src * sizeof (iovec_t)); - if (dst_iovecs != NULL) - kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t)); - - *enc_len = 0; - *authbuf = NULL; - *auth_len = 0; - *no_crypt = B_FALSE; - puio->uio_iov = NULL; - puio->uio_iovcnt = 0; - cuio->uio_iov = NULL; - cuio->uio_iovcnt = 0; - return (ret); -} - -/* - * Special case handling routine for encrypting / decrypting dnode blocks. - */ -static int -zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version, - uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, - zfs_uio_t *puio, zfs_uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, - uint_t *auth_len, boolean_t *no_crypt) -{ - int ret; - uint_t nr_src, nr_dst, crypt_len; - uint_t aad_len = 0, nr_iovecs = 0, total_len = 0; - uint_t i, j, max_dnp = datalen >> DNODE_SHIFT; - iovec_t *src_iovecs = NULL, *dst_iovecs = NULL; - uint8_t *src, *dst, *aadp; - dnode_phys_t *dnp, *adnp, *sdnp, *ddnp; - uint8_t *aadbuf = zio_buf_alloc(datalen); - - if (encrypt) { - src = plainbuf; - dst = cipherbuf; - nr_src = 0; - nr_dst = 1; - } else { - src = cipherbuf; - dst = plainbuf; - nr_src = 1; - nr_dst = 0; - } - - sdnp = (dnode_phys_t *)src; - ddnp = (dnode_phys_t *)dst; - aadp = aadbuf; - - /* - * Count the number of iovecs we will need to do the encryption by - * counting the number of bonus buffers that need to be encrypted. - */ - for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { - /* - * This block may still be byteswapped. However, all of the - * values we use are either uint8_t's (for which byteswapping - * is a noop) or a * != 0 check, which will work regardless - * of whether or not we byteswap. - */ - if (sdnp[i].dn_type != DMU_OT_NONE && - DMU_OT_IS_ENCRYPTED(sdnp[i].dn_bonustype) && - sdnp[i].dn_bonuslen != 0) { - nr_iovecs++; - } - } - - nr_src += nr_iovecs; - nr_dst += nr_iovecs; - - if (nr_src != 0) { - src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP); - if (src_iovecs == NULL) { - ret = SET_ERROR(ENOMEM); - goto error; - } - } - - if (nr_dst != 0) { - dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP); - if (dst_iovecs == NULL) { - ret = SET_ERROR(ENOMEM); - goto error; - } - } - - nr_iovecs = 0; - - /* - * Iterate through the dnodes again, this time filling in the uios - * we allocated earlier. We also concatenate any data we want to - * authenticate onto aadbuf. - */ - for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { - dnp = &sdnp[i]; - - /* copy over the core fields and blkptrs (kept as plaintext) */ - memcpy(&ddnp[i], dnp, - (uint8_t *)DN_BONUS(dnp) - (uint8_t *)dnp); - - if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { - memcpy(DN_SPILL_BLKPTR(&ddnp[i]), DN_SPILL_BLKPTR(dnp), - sizeof (blkptr_t)); - } - - /* - * Handle authenticated data. We authenticate everything in - * the dnode that can be brought over when we do a raw send. - * This includes all of the core fields as well as the MACs - * stored in the bp checksums and all of the portable bits - * from blk_prop. We include the dnode padding here in case it - * ever gets used in the future. Some dn_flags and dn_used are - * not portable so we mask those out values out of the - * authenticated data. - */ - crypt_len = offsetof(dnode_phys_t, dn_blkptr); - memcpy(aadp, dnp, crypt_len); - adnp = (dnode_phys_t *)aadp; - adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK; - adnp->dn_used = 0; - aadp += crypt_len; - aad_len += crypt_len; - - for (j = 0; j < dnp->dn_nblkptr; j++) { - zio_crypt_bp_do_aad_updates(&aadp, &aad_len, - version, byteswap, &dnp->dn_blkptr[j]); - } - - if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { - zio_crypt_bp_do_aad_updates(&aadp, &aad_len, - version, byteswap, DN_SPILL_BLKPTR(dnp)); - } - - /* - * If this bonus buffer needs to be encrypted, we prepare an - * iovec_t. The encryption / decryption functions will fill - * this in for us with the encrypted or decrypted data. - * Otherwise we add the bonus buffer to the authenticated - * data buffer and copy it over to the destination. The - * encrypted iovec extends to DN_MAX_BONUS_LEN(dnp) so that - * we can guarantee alignment with the AES block size - * (128 bits). - */ - crypt_len = DN_MAX_BONUS_LEN(dnp); - if (dnp->dn_type != DMU_OT_NONE && - DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) && - dnp->dn_bonuslen != 0) { - ASSERT3U(nr_iovecs, <, nr_src); - ASSERT3U(nr_iovecs, <, nr_dst); - ASSERT3P(src_iovecs, !=, NULL); - ASSERT3P(dst_iovecs, !=, NULL); - src_iovecs[nr_iovecs].iov_base = DN_BONUS(dnp); - src_iovecs[nr_iovecs].iov_len = crypt_len; - dst_iovecs[nr_iovecs].iov_base = DN_BONUS(&ddnp[i]); - dst_iovecs[nr_iovecs].iov_len = crypt_len; - - nr_iovecs++; - total_len += crypt_len; - } else { - memcpy(DN_BONUS(&ddnp[i]), DN_BONUS(dnp), crypt_len); - memcpy(aadp, DN_BONUS(dnp), crypt_len); - aadp += crypt_len; - aad_len += crypt_len; - } - } - - *no_crypt = (nr_iovecs == 0); - *enc_len = total_len; - *authbuf = aadbuf; - *auth_len = aad_len; - - if (encrypt) { - puio->uio_iov = src_iovecs; - puio->uio_iovcnt = nr_src; - cuio->uio_iov = dst_iovecs; - cuio->uio_iovcnt = nr_dst; - } else { - puio->uio_iov = dst_iovecs; - puio->uio_iovcnt = nr_dst; - cuio->uio_iov = src_iovecs; - cuio->uio_iovcnt = nr_src; - } - - return (0); - -error: - zio_buf_free(aadbuf, datalen); - if (src_iovecs != NULL) - kmem_free(src_iovecs, nr_src * sizeof (iovec_t)); - if (dst_iovecs != NULL) - kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t)); - - *enc_len = 0; - *authbuf = NULL; - *auth_len = 0; - *no_crypt = B_FALSE; - puio->uio_iov = NULL; - puio->uio_iovcnt = 0; - cuio->uio_iov = NULL; - cuio->uio_iovcnt = 0; - return (ret); -} - -static int -zio_crypt_init_uios_normal(boolean_t encrypt, uint8_t *plainbuf, - uint8_t *cipherbuf, uint_t datalen, zfs_uio_t *puio, zfs_uio_t *cuio, - uint_t *enc_len) -{ - (void) encrypt; - int ret; - uint_t nr_plain = 1, nr_cipher = 2; - iovec_t *plain_iovecs = NULL, *cipher_iovecs = NULL; - - /* allocate the iovecs for the plain and cipher data */ - plain_iovecs = kmem_alloc(nr_plain * sizeof (iovec_t), - KM_SLEEP); - if (!plain_iovecs) { - ret = SET_ERROR(ENOMEM); - goto error; - } - - cipher_iovecs = kmem_alloc(nr_cipher * sizeof (iovec_t), - KM_SLEEP); - if (!cipher_iovecs) { - ret = SET_ERROR(ENOMEM); - goto error; - } - - plain_iovecs[0].iov_base = plainbuf; - plain_iovecs[0].iov_len = datalen; - cipher_iovecs[0].iov_base = cipherbuf; - cipher_iovecs[0].iov_len = datalen; - - *enc_len = datalen; - puio->uio_iov = plain_iovecs; - puio->uio_iovcnt = nr_plain; - cuio->uio_iov = cipher_iovecs; - cuio->uio_iovcnt = nr_cipher; - - return (0); - -error: - if (plain_iovecs != NULL) - kmem_free(plain_iovecs, nr_plain * sizeof (iovec_t)); - if (cipher_iovecs != NULL) - kmem_free(cipher_iovecs, nr_cipher * sizeof (iovec_t)); - - *enc_len = 0; - puio->uio_iov = NULL; - puio->uio_iovcnt = 0; - cuio->uio_iov = NULL; - cuio->uio_iovcnt = 0; - return (ret); -} - -/* - * This function builds up the plaintext (puio) and ciphertext (cuio) uios so - * that they can be used for encryption and decryption by zio_do_crypt_uio(). - * Most blocks will use zio_crypt_init_uios_normal(), with ZIL and dnode blocks - * requiring special handling to parse out pieces that are to be encrypted. The - * authbuf is used by these special cases to store additional authenticated - * data (AAD) for the encryption modes. - */ -static int -zio_crypt_init_uios(boolean_t encrypt, uint64_t version, dmu_object_type_t ot, - uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, - uint8_t *mac, zfs_uio_t *puio, zfs_uio_t *cuio, uint_t *enc_len, - uint8_t **authbuf, uint_t *auth_len, boolean_t *no_crypt) -{ - int ret; - iovec_t *mac_iov; - - ASSERT(DMU_OT_IS_ENCRYPTED(ot) || ot == DMU_OT_NONE); - - /* route to handler */ - switch (ot) { - case DMU_OT_INTENT_LOG: - ret = zio_crypt_init_uios_zil(encrypt, plainbuf, cipherbuf, - datalen, byteswap, puio, cuio, enc_len, authbuf, auth_len, - no_crypt); - break; - case DMU_OT_DNODE: - ret = zio_crypt_init_uios_dnode(encrypt, version, plainbuf, - cipherbuf, datalen, byteswap, puio, cuio, enc_len, authbuf, - auth_len, no_crypt); - break; - default: - ret = zio_crypt_init_uios_normal(encrypt, plainbuf, cipherbuf, - datalen, puio, cuio, enc_len); - *authbuf = NULL; - *auth_len = 0; - *no_crypt = B_FALSE; - break; - } - - if (ret != 0) - goto error; - - /* populate the uios */ - puio->uio_segflg = UIO_SYSSPACE; - cuio->uio_segflg = UIO_SYSSPACE; - - mac_iov = ((iovec_t *)&cuio->uio_iov[cuio->uio_iovcnt - 1]); - mac_iov->iov_base = mac; - mac_iov->iov_len = ZIO_DATA_MAC_LEN; - - return (0); - -error: - return (ret); -} - -/* - * Primary encryption / decryption entrypoint for zio data. - */ -int -zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, - dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv, - uint8_t *mac, uint_t datalen, uint8_t *plainbuf, uint8_t *cipherbuf, - boolean_t *no_crypt) -{ - int ret; - boolean_t locked = B_FALSE; - uint64_t crypt = key->zk_crypt; - uint_t keydata_len = zio_crypt_table[crypt].ci_keylen; - uint_t enc_len, auth_len; - zfs_uio_t puio, cuio; - uint8_t enc_keydata[MASTER_KEY_MAX_LEN]; - crypto_key_t tmp_ckey, *ckey = NULL; - crypto_ctx_template_t tmpl; - uint8_t *authbuf = NULL; - - memset(&puio, 0, sizeof (puio)); - memset(&cuio, 0, sizeof (cuio)); - - /* - * If the needed key is the current one, just use it. Otherwise we - * need to generate a temporary one from the given salt + master key. - * If we are encrypting, we must return a copy of the current salt - * so that it can be stored in the blkptr_t. - */ - rw_enter(&key->zk_salt_lock, RW_READER); - locked = B_TRUE; - - if (memcmp(salt, key->zk_salt, ZIO_DATA_SALT_LEN) == 0) { - ckey = &key->zk_current_key; - tmpl = key->zk_current_tmpl; - } else { - rw_exit(&key->zk_salt_lock); - locked = B_FALSE; - - ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, - salt, ZIO_DATA_SALT_LEN, enc_keydata, keydata_len); - if (ret != 0) - goto error; - - tmp_ckey.ck_data = enc_keydata; - tmp_ckey.ck_length = CRYPTO_BYTES2BITS(keydata_len); - - ckey = &tmp_ckey; - tmpl = NULL; - } - - /* - * Attempt to use QAT acceleration if we can. We currently don't - * do this for metadnode and ZIL blocks, since they have a much - * more involved buffer layout and the qat_crypt() function only - * works in-place. - */ - if (qat_crypt_use_accel(datalen) && - ot != DMU_OT_INTENT_LOG && ot != DMU_OT_DNODE) { - uint8_t *srcbuf, *dstbuf; - - if (encrypt) { - srcbuf = plainbuf; - dstbuf = cipherbuf; - } else { - srcbuf = cipherbuf; - dstbuf = plainbuf; - } - - ret = qat_crypt((encrypt) ? QAT_ENCRYPT : QAT_DECRYPT, srcbuf, - dstbuf, NULL, 0, iv, mac, ckey, key->zk_crypt, datalen); - if (ret == CPA_STATUS_SUCCESS) { - if (locked) { - rw_exit(&key->zk_salt_lock); - locked = B_FALSE; - } - - return (0); - } - /* If the hardware implementation fails fall back to software */ - } - - /* create uios for encryption */ - ret = zio_crypt_init_uios(encrypt, key->zk_version, ot, plainbuf, - cipherbuf, datalen, byteswap, mac, &puio, &cuio, &enc_len, - &authbuf, &auth_len, no_crypt); - if (ret != 0) - goto error; - - /* perform the encryption / decryption in software */ - ret = zio_do_crypt_uio(encrypt, key->zk_crypt, ckey, tmpl, iv, enc_len, - &puio, &cuio, authbuf, auth_len); - if (ret != 0) - goto error; - - if (locked) { - rw_exit(&key->zk_salt_lock); - } - - if (authbuf != NULL) - zio_buf_free(authbuf, datalen); - if (ckey == &tmp_ckey) - memset(enc_keydata, 0, keydata_len); - zio_crypt_destroy_uio(&puio); - zio_crypt_destroy_uio(&cuio); - - return (0); - -error: - if (locked) - rw_exit(&key->zk_salt_lock); - if (authbuf != NULL) - zio_buf_free(authbuf, datalen); - if (ckey == &tmp_ckey) - memset(enc_keydata, 0, keydata_len); - zio_crypt_destroy_uio(&puio); - zio_crypt_destroy_uio(&cuio); - - return (ret); -} - -/* - * Simple wrapper around zio_do_crypt_data() to work with abd's instead of - * linear buffers. - */ -int -zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot, - boolean_t byteswap, uint8_t *salt, uint8_t *iv, uint8_t *mac, - uint_t datalen, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt) -{ - int ret; - void *ptmp, *ctmp; - - if (encrypt) { - ptmp = abd_borrow_buf_copy(pabd, datalen); - ctmp = abd_borrow_buf(cabd, datalen); - } else { - ptmp = abd_borrow_buf(pabd, datalen); - ctmp = abd_borrow_buf_copy(cabd, datalen); - } - - ret = zio_do_crypt_data(encrypt, key, ot, byteswap, salt, iv, mac, - datalen, ptmp, ctmp, no_crypt); - if (ret != 0) - goto error; - - if (encrypt) { - abd_return_buf(pabd, ptmp, datalen); - abd_return_buf_copy(cabd, ctmp, datalen); - } else { - abd_return_buf_copy(pabd, ptmp, datalen); - abd_return_buf(cabd, ctmp, datalen); - } - - return (0); - -error: - if (encrypt) { - abd_return_buf(pabd, ptmp, datalen); - abd_return_buf_copy(cabd, ctmp, datalen); - } else { - abd_return_buf_copy(pabd, ptmp, datalen); - abd_return_buf(cabd, ctmp, datalen); - } - - return (ret); -} - -#if defined(_KERNEL) -module_param(zfs_key_max_salt_uses, ulong, 0644); -MODULE_PARM_DESC(zfs_key_max_salt_uses, "Max number of times a salt value " - "can be used for generating encryption keys before it is rotated"); -#endif diff --git a/module/os/freebsd/zfs/zio_crypt.c b/module/zfs/zio_crypt.c similarity index 76% rename from module/os/freebsd/zfs/zio_crypt.c rename to module/zfs/zio_crypt.c index 49d5b2cedb31..a205fac61296 100644 --- a/module/os/freebsd/zfs/zio_crypt.c +++ b/module/zfs/zio_crypt.c @@ -16,6 +16,7 @@ /* * Copyright (c) 2017, Datto, Inc. All rights reserved. + * Copyright (c) 2026, TrueNAS. */ #include @@ -27,6 +28,7 @@ #include #include #include +#include /* * This file is responsible for handling all of the details of generating @@ -186,7 +188,7 @@ #define ZFS_KEY_MAX_SALT_USES_DEFAULT 400000000 #define ZFS_CURRENT_MAX_SALT_USES \ (MIN(zfs_key_max_salt_uses, ZFS_KEY_MAX_SALT_USES_DEFAULT)) -static unsigned long zfs_key_max_salt_uses = ZFS_KEY_MAX_SALT_USES_DEFAULT; +uint64_t zfs_key_max_salt_uses = ZFS_KEY_MAX_SALT_USES_DEFAULT; typedef struct blkptr_auth_buf { uint64_t bab_prop; /* blk_prop - portable mask */ @@ -206,96 +208,73 @@ const zio_crypt_info_t zio_crypt_table[ZIO_CRYPT_FUNCTIONS] = { {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 32, "aes-256-gcm"} }; -static void -zio_crypt_key_destroy_early(zio_crypt_key_t *key) +void +zio_crypt_key_destroy(zio_crypt_key_t *key) { - rw_destroy(&key->zk_salt_lock); + zio_crypt_key_close_os(key); - /* free crypto templates */ - memset(&key->zk_session, 0, sizeof (key->zk_session)); + rw_destroy(&key->zk_salt_lock); /* zero out sensitive data */ memset(key, 0, sizeof (zio_crypt_key_t)); } -void -zio_crypt_key_destroy(zio_crypt_key_t *key) -{ - - freebsd_crypt_freesession(&key->zk_session); - zio_crypt_key_destroy_early(key); -} - int zio_crypt_key_init(uint64_t crypt, zio_crypt_key_t *key) { int ret; - crypto_mechanism_t mech __unused; - uint_t keydata_len; - const zio_crypt_info_t *ci = NULL; ASSERT3P(key, !=, NULL); ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); - ci = &zio_crypt_table[crypt]; + const zio_crypt_info_t *ci = &zio_crypt_table[crypt]; if (ci->ci_crypt_type != ZC_TYPE_GCM && ci->ci_crypt_type != ZC_TYPE_CCM) return (ENOTSUP); - keydata_len = zio_crypt_table[crypt].ci_keylen; memset(key, 0, sizeof (zio_crypt_key_t)); - rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); /* fill keydata buffers and salt with random data */ ret = random_get_bytes((uint8_t *)&key->zk_guid, sizeof (uint64_t)); if (ret != 0) - goto error; + return (ret); - ret = random_get_bytes(key->zk_master_keydata, keydata_len); + ret = random_get_bytes(key->zk_master_keydata, ci->ci_keylen); if (ret != 0) - goto error; + return (ret); ret = random_get_bytes(key->zk_hmac_keydata, SHA512_HMAC_KEYLEN); if (ret != 0) - goto error; + return (ret); ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); if (ret != 0) - goto error; + return (ret); /* derive the current key from the master key */ - ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, + ret = hkdf_sha512(key->zk_master_keydata, ci->ci_keylen, NULL, 0, key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, - keydata_len); + ci->ci_keylen); if (ret != 0) - goto error; + return (ret); /* initialize keys for the ICP */ key->zk_current_key.ck_data = key->zk_current_keydata; - key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); + key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(ci->ci_keylen); key->zk_hmac_key.ck_data = &key->zk_hmac_key; key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); - ci = &zio_crypt_table[crypt]; - if (ci->ci_crypt_type != ZC_TYPE_GCM && - ci->ci_crypt_type != ZC_TYPE_CCM) - return (ENOTSUP); - - ret = freebsd_crypt_newsession(&key->zk_session, ci, - &key->zk_current_key); - if (ret) - goto error; + ret = zio_crypt_key_open_os(key, ci); + if (ret != 0) + return (ret); + rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); key->zk_crypt = crypt; key->zk_version = ZIO_CRYPT_KEY_CURRENT_VERSION; key->zk_salt_count = 0; return (0); - -error: - zio_crypt_key_destroy_early(key); - return (ret); } static int @@ -303,9 +282,7 @@ zio_crypt_key_change_salt(zio_crypt_key_t *key) { int ret = 0; uint8_t salt[ZIO_DATA_SALT_LEN]; - crypto_mechanism_t mech __unused; - - uint_t keydata_len = zio_crypt_table[key->zk_crypt].ci_keylen; + const zio_crypt_info_t *ci = &zio_crypt_table[key->zk_crypt]; /* generate a new salt */ ret = random_get_bytes(salt, ZIO_DATA_SALT_LEN); @@ -319,8 +296,8 @@ zio_crypt_key_change_salt(zio_crypt_key_t *key) goto out_unlock; /* derive the current key from the master key and the new salt */ - ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, - salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, keydata_len); + ret = hkdf_sha512(key->zk_master_keydata, ci->ci_keylen, NULL, 0, + salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, ci->ci_keylen); if (ret != 0) goto out_unlock; @@ -328,9 +305,7 @@ zio_crypt_key_change_salt(zio_crypt_key_t *key) memcpy(key->zk_salt, salt, ZIO_DATA_SALT_LEN); key->zk_salt_count = 0; - freebsd_crypt_freesession(&key->zk_session); - ret = freebsd_crypt_newsession(&key->zk_session, - &zio_crypt_table[key->zk_crypt], &key->zk_current_key); + ret = zio_crypt_key_reopen_os(key, ci); if (ret != 0) goto out_unlock; @@ -371,92 +346,19 @@ zio_crypt_key_get_salt(zio_crypt_key_t *key, uint8_t *salt) return (ret); } -/* - * This function handles all encryption and decryption in zfs. When - * encrypting it expects puio to reference the plaintext and cuio to - * reference the ciphertext. cuio must have enough space for the - * ciphertext + room for a MAC. datalen should be the length of the - * plaintext / ciphertext alone. - */ -/* - * The implementation for FreeBSD's OpenCrypto. - * - * The big difference between ICP and FOC is that FOC uses a single - * buffer for input and output. This means that (for AES-GCM, the - * only one supported right now) the source must be copied into the - * destination, and the destination must have the AAD, and the tag/MAC, - * already associated with it. (Both implementations can use a uio.) - * - * Since the auth data is part of the iovec array, all we need to know - * is the length: 0 means there's no AAD. - * - */ -static int -zio_do_crypt_uio_opencrypto(boolean_t encrypt, freebsd_crypt_session_t *sess, - uint64_t crypt, crypto_key_t *key, uint8_t *ivbuf, uint_t datalen, - zfs_uio_t *uio, uint_t auth_len) -{ - const zio_crypt_info_t *ci = &zio_crypt_table[crypt]; - if (ci->ci_crypt_type != ZC_TYPE_GCM && - ci->ci_crypt_type != ZC_TYPE_CCM) - return (ENOTSUP); - - - int ret = freebsd_crypt_uio(encrypt, sess, ci, uio, key, ivbuf, - datalen, auth_len); - if (ret != 0) { -#ifdef FCRYPTO_DEBUG - printf("%s(%d): Returning error %s\n", - __FUNCTION__, __LINE__, encrypt ? "EIO" : "ECKSUM"); -#endif - ret = SET_ERROR(encrypt ? EIO : ECKSUM); - } - - return (ret); -} - int zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv, uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out) { int ret; uint64_t aad[3]; - /* - * With OpenCrypto in FreeBSD, the same buffer is used for - * input and output. Also, the AAD (for AES-GMC at least) - * needs to logically go in front. - */ - zfs_uio_t cuio; - struct uio cuio_s; - iovec_t iovecs[4]; - uint64_t crypt = key->zk_crypt; - uint_t enc_len, keydata_len, aad_len; - - ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); - - memset(&cuio_s, 0, sizeof (cuio_s)); - zfs_uio_init(&cuio, &cuio_s); - - keydata_len = zio_crypt_table[crypt].ci_keylen; + uint_t aad_len; + const zio_crypt_info_t *ci = &zio_crypt_table[key->zk_crypt]; /* generate iv for wrapping the master and hmac key */ ret = random_get_pseudo_bytes(iv, WRAPPING_IV_LEN); if (ret != 0) - goto error; - - /* - * Since we only support one buffer, we need to copy - * the plain text (source) to the cipher buffer (dest). - * We set iovecs[0] -- the authentication data -- below. - */ - memcpy(keydata_out, key->zk_master_keydata, keydata_len); - memcpy(hmac_keydata_out, key->zk_hmac_keydata, SHA512_HMAC_KEYLEN); - iovecs[1].iov_base = keydata_out; - iovecs[1].iov_len = keydata_len; - iovecs[2].iov_base = hmac_keydata_out; - iovecs[2].iov_len = SHA512_HMAC_KEYLEN; - iovecs[3].iov_base = mac; - iovecs[3].iov_len = WRAPPING_MAC_LEN; + return (ret); /* * Although we don't support writing to the old format, we do @@ -470,27 +372,33 @@ zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv, ASSERT3U(key->zk_version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); aad_len = sizeof (uint64_t) * 3; aad[0] = LE_64(key->zk_guid); - aad[1] = LE_64(crypt); + aad[1] = LE_64(key->zk_crypt); aad[2] = LE_64(key->zk_version); } - iovecs[0].iov_base = aad; - iovecs[0].iov_len = aad_len; - enc_len = zio_crypt_table[crypt].ci_keylen + SHA512_HMAC_KEYLEN; + zfs_uio_t puio, cuio; + int idx; + ret = zio_crypt_uios_init_os(&puio, &cuio, 2, &idx); + if (ret != 0) + return (ret); + + zfs_uio_iovbase(&puio, idx) = key->zk_master_keydata; + zfs_uio_iovlen(&puio, idx) = ci->ci_keylen; + zfs_uio_iovbase(&puio, idx+1) = key->zk_hmac_keydata; + zfs_uio_iovlen(&puio, idx+1) = SHA512_HMAC_KEYLEN; - GET_UIO_STRUCT(&cuio)->uio_iov = iovecs; - zfs_uio_iovcnt(&cuio) = 4; - zfs_uio_segflg(&cuio) = UIO_SYSSPACE; + zfs_uio_iovbase(&cuio, idx) = keydata_out; + zfs_uio_iovlen(&cuio, idx) = ci->ci_keylen; + zfs_uio_iovbase(&cuio, idx+1) = hmac_keydata_out; + zfs_uio_iovlen(&cuio, idx+1) = SHA512_HMAC_KEYLEN; /* encrypt the keys and store the resulting ciphertext and mac */ - ret = zio_do_crypt_uio_opencrypto(B_TRUE, NULL, crypt, cwkey, - iv, enc_len, &cuio, aad_len); - if (ret != 0) - goto error; + ret = zio_encrypt_os(ci, cwkey, NULL, + &puio, &cuio, ci->ci_keylen + SHA512_HMAC_KEYLEN, + iv, (uint8_t *)aad, aad_len, mac); - return (0); + zio_crypt_uios_fini_os(&puio, &cuio); -error: return (ret); } @@ -501,46 +409,11 @@ zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version, { int ret; uint64_t aad[3]; - /* - * With OpenCrypto in FreeBSD, the same buffer is used for - * input and output. Also, the AAD (for AES-GMC at least) - * needs to logically go in front. - */ - zfs_uio_t cuio; - struct uio cuio_s; - iovec_t iovecs[4]; - void *src, *dst; - uint_t enc_len, keydata_len, aad_len; + uint_t aad_len; + const zio_crypt_info_t *ci = &zio_crypt_table[crypt]; ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); - keydata_len = zio_crypt_table[crypt].ci_keylen; - rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); - - memset(&cuio_s, 0, sizeof (cuio_s)); - zfs_uio_init(&cuio, &cuio_s); - - /* - * Since we only support one buffer, we need to copy - * the encrypted buffer (source) to the plain buffer - * (dest). We set iovecs[0] -- the authentication data -- - * below. - */ - dst = key->zk_master_keydata; - src = keydata; - memcpy(dst, src, keydata_len); - - dst = key->zk_hmac_keydata; - src = hmac_keydata; - memcpy(dst, src, SHA512_HMAC_KEYLEN); - - iovecs[1].iov_base = key->zk_master_keydata; - iovecs[1].iov_len = keydata_len; - iovecs[2].iov_base = key->zk_hmac_keydata; - iovecs[2].iov_len = SHA512_HMAC_KEYLEN; - iovecs[3].iov_base = mac; - iovecs[3].iov_len = WRAPPING_MAC_LEN; - if (version == 0) { aad_len = sizeof (uint64_t); aad[0] = LE_64(guid); @@ -552,45 +425,54 @@ zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version, aad[2] = LE_64(version); } - enc_len = keydata_len + SHA512_HMAC_KEYLEN; - iovecs[0].iov_base = aad; - iovecs[0].iov_len = aad_len; + zfs_uio_t cuio, puio; + int idx; + ret = zio_crypt_uios_init_os(&cuio, &puio, 2, &idx); + if (ret != 0) + goto error; - GET_UIO_STRUCT(&cuio)->uio_iov = iovecs; - zfs_uio_iovcnt(&cuio) = 4; - zfs_uio_segflg(&cuio) = UIO_SYSSPACE; + zfs_uio_iovbase(&cuio, idx) = keydata; + zfs_uio_iovlen(&cuio, idx) = ci->ci_keylen; + zfs_uio_iovbase(&cuio, idx+1) = hmac_keydata; + zfs_uio_iovlen(&cuio, idx+1) = SHA512_HMAC_KEYLEN; - /* decrypt the keys and store the result in the output buffers */ - ret = zio_do_crypt_uio_opencrypto(B_FALSE, NULL, crypt, cwkey, - iv, enc_len, &cuio, aad_len); + zfs_uio_iovbase(&puio, idx) = key->zk_master_keydata; + zfs_uio_iovlen(&puio, idx) = ci->ci_keylen; + zfs_uio_iovbase(&puio, idx+1) = key->zk_hmac_keydata; + zfs_uio_iovlen(&puio, idx+1) = SHA512_HMAC_KEYLEN; + ret = zio_decrypt_os(ci, cwkey, NULL, + &cuio, &puio, ci->ci_keylen + SHA512_HMAC_KEYLEN, + iv, (uint8_t *)aad, aad_len, mac); if (ret != 0) goto error; + zio_crypt_uios_fini_os(&cuio, &puio); + /* generate a fresh salt */ ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); if (ret != 0) goto error; /* derive the current key from the master key */ - ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, + ret = hkdf_sha512(key->zk_master_keydata, ci->ci_keylen, NULL, 0, key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, - keydata_len); + ci->ci_keylen); if (ret != 0) goto error; /* initialize keys for ICP */ key->zk_current_key.ck_data = key->zk_current_keydata; - key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); + key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(ci->ci_keylen); key->zk_hmac_key.ck_data = key->zk_hmac_keydata; key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); - ret = freebsd_crypt_newsession(&key->zk_session, - &zio_crypt_table[crypt], &key->zk_current_key); + ret = zio_crypt_key_open_os(key, ci); if (ret != 0) goto error; + rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); key->zk_crypt = crypt; key->zk_version = version; key->zk_guid = guid; @@ -599,7 +481,7 @@ zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version, return (0); error: - zio_crypt_key_destroy_early(key); + memset(key, 0, sizeof (zio_crypt_key_t)); return (ret); } @@ -624,16 +506,17 @@ int zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen, uint8_t *digestbuf, uint_t digestlen) { - uint8_t raw_digestbuf[SHA512_DIGEST_LENGTH]; - - ASSERT3U(digestlen, <=, SHA512_DIGEST_LENGTH); + uint8_t raw_digestbuf[SHA512_HMAC_LEN]; - crypto_mac(&key->zk_hmac_key, data, datalen, - raw_digestbuf, SHA512_DIGEST_LENGTH); + ASSERT3U(digestlen, <=, SHA512_HMAC_LEN); - memcpy(digestbuf, raw_digestbuf, digestlen); + int err = zio_crypt_hmac_os(key, data, datalen, raw_digestbuf); + if (err == 0) + memcpy(digestbuf, raw_digestbuf, digestlen); + else + memset(digestbuf, 0, digestlen); - return (0); + return (err); } int @@ -641,10 +524,9 @@ zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t *key, uint8_t *data, uint_t datalen, uint8_t *ivbuf, uint8_t *salt) { int ret; - uint8_t digestbuf[SHA512_DIGEST_LENGTH]; + uint8_t digestbuf[SHA512_HMAC_LEN]; - ret = zio_crypt_do_hmac(key, data, datalen, - digestbuf, SHA512_DIGEST_LENGTH); + ret = zio_crypt_hmac_os(key, data, datalen, digestbuf); if (ret != 0) return (ret); @@ -831,7 +713,6 @@ zio_crypt_copy_dnode_bonus(abd_t *src_abd, uint8_t *dst, uint_t datalen) static void zio_crypt_bp_zero_nonportable_blkprop(blkptr_t *bp, uint64_t version) { - int avoidlint = SPA_MINBLOCKSIZE; /* * Version 0 did not properly zero out all non-portable fields * as it should have done. We maintain this code so that we can @@ -840,7 +721,7 @@ zio_crypt_bp_zero_nonportable_blkprop(blkptr_t *bp, uint64_t version) if (version == 0) { BP_SET_DEDUP(bp, 0); BP_SET_CHECKSUM(bp, 0); - BP_SET_PSIZE(bp, avoidlint); + BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE); return; } @@ -876,7 +757,7 @@ zio_crypt_bp_zero_nonportable_blkprop(blkptr_t *bp, uint64_t version) * asserts, but the value doesn't really matter as * long as it is constant. */ - BP_SET_PSIZE(bp, avoidlint); + BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE); } BP_SET_DEDUP(bp, 0); @@ -913,16 +794,14 @@ zio_crypt_bp_auth_init(uint64_t version, boolean_t should_bswap, blkptr_t *bp, } static int -zio_crypt_bp_do_hmac_updates(crypto_context_t ctx, uint64_t version, +zio_crypt_bp_do_hmac_updates(zio_crypt_hmac_t *hmac, uint64_t version, boolean_t should_bswap, blkptr_t *bp) { uint_t bab_len; blkptr_auth_buf_t bab; zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); - crypto_mac_update(ctx, &bab, bab_len); - - return (0); + return (zio_crypt_hmac_update_os(hmac, (uint8_t *)&bab, bab_len)); } static void @@ -950,17 +829,23 @@ zio_crypt_bp_do_aad_updates(uint8_t **aadp, uint_t *aad_len, uint64_t version, } static int -zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx, uint64_t version, +zio_crypt_do_dnode_hmac_updates(zio_crypt_hmac_t *hmac, uint64_t version, boolean_t should_bswap, dnode_phys_t *dnp) { int ret, i; - dnode_phys_t *adnp; + dnode_phys_t *adnp, tmp_dncore; + size_t dn_core_size = offsetof(dnode_phys_t, dn_blkptr); boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); - uint8_t tmp_dncore[offsetof(dnode_phys_t, dn_blkptr)]; - /* authenticate the core dnode (masking out non-portable bits) */ - memcpy(tmp_dncore, dnp, sizeof (tmp_dncore)); - adnp = (dnode_phys_t *)tmp_dncore; + /* + * Authenticate the core dnode (masking out non-portable bits). + * We only copy the first 64 bytes we operate on to avoid the overhead + * of copying 512-64 unneeded bytes. The compiler seems to be fine + * with that. + */ + memcpy(&tmp_dncore, dnp, dn_core_size); + adnp = &tmp_dncore; + if (le_bswap) { adnp->dn_datablkszsec = BSWAP_16(adnp->dn_datablkszsec); adnp->dn_bonuslen = BSWAP_16(adnp->dn_bonuslen); @@ -970,17 +855,19 @@ zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx, uint64_t version, adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK; adnp->dn_used = 0; - crypto_mac_update(ctx, adnp, sizeof (tmp_dncore)); + ret = zio_crypt_hmac_update_os(hmac, (uint8_t *)adnp, dn_core_size); + if (ret != 0) + goto error; for (i = 0; i < dnp->dn_nblkptr; i++) { - ret = zio_crypt_bp_do_hmac_updates(ctx, version, + ret = zio_crypt_bp_do_hmac_updates(hmac, version, should_bswap, &dnp->dn_blkptr[i]); if (ret != 0) goto error; } if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { - ret = zio_crypt_bp_do_hmac_updates(ctx, version, + ret = zio_crypt_bp_do_hmac_updates(hmac, version, should_bswap, DN_SPILL_BLKPTR(dnp)); if (ret != 0) goto error; @@ -1020,22 +907,25 @@ int zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen, boolean_t should_bswap, uint8_t *portable_mac, uint8_t *local_mac) { - int ret; - struct hmac_ctx hash_ctx; - struct hmac_ctx *ctx = &hash_ctx; + int err; + zio_crypt_hmac_t hmac; objset_phys_t *osp = data; uint64_t intval; boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); - uint8_t raw_portable_mac[SHA512_DIGEST_LENGTH]; - uint8_t raw_local_mac[SHA512_DIGEST_LENGTH]; - + uint8_t raw_portable_mac[SHA512_HMAC_LEN]; + uint8_t raw_local_mac[SHA512_HMAC_LEN]; /* calculate the portable MAC from the portable fields and metadnode */ - crypto_mac_init(ctx, &key->zk_hmac_key); + err = zio_crypt_hmac_init_os(&hmac, key); + if (err) + goto error; /* add in the os_type */ intval = (le_bswap) ? osp->os_type : BSWAP_64(osp->os_type); - crypto_mac_update(ctx, &intval, sizeof (uint64_t)); + err = zio_crypt_hmac_update_os(&hmac, (const uint8_t *)&intval, + sizeof (intval)); + if (err) + goto error; /* add in the portable os_flags */ intval = osp->os_flags; @@ -1045,15 +935,21 @@ zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen, if (!ZFS_HOST_BYTEORDER) intval = BSWAP_64(intval); - crypto_mac_update(ctx, &intval, sizeof (uint64_t)); + err = zio_crypt_hmac_update_os(&hmac, (const uint8_t *)&intval, + sizeof (intval)); + if (err) + goto error; /* add in fields from the metadnode */ - ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, + err = zio_crypt_do_dnode_hmac_updates(&hmac, key->zk_version, should_bswap, &osp->os_meta_dnode); - if (ret) + if (err) goto error; - crypto_mac_final(ctx, raw_portable_mac, SHA512_DIGEST_LENGTH); + /* store the final digest in a temporary buffer and copy what we need */ + err = zio_crypt_hmac_final_os(&hmac, raw_portable_mac); + if (err) + goto error; memcpy(portable_mac, raw_portable_mac, ZIO_OBJSET_MAC_LEN); @@ -1089,7 +985,9 @@ zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen, } /* calculate the local MAC from the userused and groupused dnodes */ - crypto_mac_init(ctx, &key->zk_hmac_key); + err = zio_crypt_hmac_init_os(&hmac, key); + if (err) + goto error; /* add in the non-portable os_flags */ intval = osp->os_flags; @@ -1099,33 +997,38 @@ zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen, if (!ZFS_HOST_BYTEORDER) intval = BSWAP_64(intval); - crypto_mac_update(ctx, &intval, sizeof (uint64_t)); + err = zio_crypt_hmac_update_os(&hmac, (const uint8_t *)&intval, + sizeof (intval)); + if (err) + goto error; - /* XXX check dnode type ... */ /* add in fields from the user accounting dnodes */ if (osp->os_userused_dnode.dn_type != DMU_OT_NONE) { - ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, + err = zio_crypt_do_dnode_hmac_updates(&hmac, key->zk_version, should_bswap, &osp->os_userused_dnode); - if (ret) + if (err) goto error; } if (osp->os_groupused_dnode.dn_type != DMU_OT_NONE) { - ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, + err = zio_crypt_do_dnode_hmac_updates(&hmac, key->zk_version, should_bswap, &osp->os_groupused_dnode); - if (ret) + if (err) goto error; } if (osp->os_projectused_dnode.dn_type != DMU_OT_NONE && datalen >= OBJSET_PHYS_SIZE_V3) { - ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, + err = zio_crypt_do_dnode_hmac_updates(&hmac, key->zk_version, should_bswap, &osp->os_projectused_dnode); - if (ret) + if (err) goto error; } - crypto_mac_final(ctx, raw_local_mac, SHA512_DIGEST_LENGTH); + /* store the final digest in a temporary buffer and copy what we need */ + err = zio_crypt_hmac_final_os(&hmac, raw_local_mac); + if (err) + goto error; memcpy(local_mac, raw_local_mac, ZIO_OBJSET_MAC_LEN); @@ -1134,15 +1037,7 @@ zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen, error: memset(portable_mac, 0, ZIO_OBJSET_MAC_LEN); memset(local_mac, 0, ZIO_OBJSET_MAC_LEN); - return (ret); -} - -static void -zio_crypt_destroy_uio(zfs_uio_t *uio) -{ - if (GET_UIO_STRUCT(uio)->uio_iov) - kmem_free(GET_UIO_STRUCT(uio)->uio_iov, - zfs_uio_iovcnt(uio) * sizeof (iovec_t)); + return (err); } /* @@ -1159,7 +1054,7 @@ zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate, void *buf, blkptr_t *bp; int i, epb = datalen >> SPA_BLKPTRSHIFT; SHA2_CTX ctx; - uint8_t digestbuf[SHA512_DIGEST_LENGTH]; + uint8_t digestbuf[SHA512_HMAC_LEN]; /* checksum all of the MACs from the layer below */ SHA2Init(SHA512, &ctx); @@ -1174,12 +1069,9 @@ zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate, void *buf, return (0); } - if (memcmp(digestbuf, cksum, ZIO_DATA_MAC_LEN) != 0) { -#ifdef FCRYPTO_DEBUG - printf("%s(%d): Setting ECKSUM\n", __FUNCTION__, __LINE__); -#endif + if (memcmp(digestbuf, cksum, ZIO_DATA_MAC_LEN) != 0) return (SET_ERROR(ECKSUM)); - } + return (0); } @@ -1230,30 +1122,20 @@ zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate, abd_t *abd, * here is to encrypt everything except the blkptr_t of a lr_write_t and * the zil_chain_t header. Everything that is not encrypted is authenticated. */ -/* - * The OpenCrypto used in FreeBSD does not use separate source and - * destination buffers; instead, the same buffer is used. Further, to - * accommodate some of the drivers, the authbuf needs to be logically before - * the data. This means that we need to copy the source to the destination, - * and set up an extra iovec_t at the beginning to handle the authbuf. - * It also means we'll only return one zfs_uio_t. - */ - static int zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, zfs_uio_t *puio, - zfs_uio_t *out_uio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len, + zfs_uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len, boolean_t *no_crypt) { - (void) puio; - uint8_t *aadbuf = zio_buf_alloc(datalen); + uint64_t txtype, lr_len, nused; + uint_t crypt_len; + uint_t aad_len = 0, nr_iovecs = 0, total_len = 0; uint8_t *src, *dst, *slrp, *dlrp, *blkend, *aadp; - iovec_t *dst_iovecs; + int idx; + iovec_t *src_iovecs, *dst_iovecs; zil_chain_t *zilc; lr_t *lr; - uint64_t txtype, lr_len, nused; - uint_t crypt_len, nr_iovecs, vec; - uint_t aad_len = 0, total_len = 0; if (encrypt) { src = plainbuf; @@ -1262,33 +1144,26 @@ zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf, src = cipherbuf; dst = plainbuf; } - memcpy(dst, src, datalen); + memset(dst, 0, datalen); - /* Find the start and end record of the log block. */ + /* find the start and end record of the log block */ zilc = (zil_chain_t *)src; slrp = src + sizeof (zil_chain_t); - aadp = aadbuf; nused = ((byteswap) ? BSWAP_64(zilc->zc_nused) : zilc->zc_nused); ASSERT3U(nused, >=, sizeof (zil_chain_t)); ASSERT3U(nused, <=, datalen); blkend = src + nused; - /* - * Calculate the number of encrypted iovecs we will need. - */ - - /* We need at least two iovecs -- one for the AAD, one for the MAC. */ - nr_iovecs = 2; - + /* calculate the number of encrypted iovecs we will need */ for (; slrp < blkend; slrp += lr_len) { lr = (lr_t *)slrp; - if (byteswap) { - txtype = BSWAP_64(lr->lrc_txtype); - lr_len = BSWAP_64(lr->lrc_reclen); - } else { + if (!byteswap) { txtype = lr->lrc_txtype; lr_len = lr->lrc_reclen; + } else { + txtype = BSWAP_64(lr->lrc_txtype); + lr_len = BSWAP_64(lr->lrc_reclen); } ASSERT3U(lr_len, >=, sizeof (lr_t)); ASSERT3U(lr_len, <=, blkend - slrp); @@ -1298,7 +1173,17 @@ zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf, nr_iovecs++; } - dst_iovecs = kmem_alloc(nr_iovecs * sizeof (iovec_t), KM_SLEEP); + int err = zio_crypt_uios_init_os(puio, cuio, nr_iovecs, &idx); + if (err != 0) + return (err); + + if (encrypt) { + src_iovecs = zfs_uio_iov(puio); + dst_iovecs = zfs_uio_iov(cuio); + } else { + src_iovecs = zfs_uio_iov(cuio); + dst_iovecs = zfs_uio_iov(puio); + } /* * Copy the plain zil header over and authenticate everything except @@ -1306,20 +1191,18 @@ zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf, * the embedded checksum will not have been calculated yet, so we don't * authenticate that. */ + uint8_t *aadbuf = zio_buf_alloc(datalen); + aadp = aadbuf; + memcpy(dst, src, sizeof (zil_chain_t)); memcpy(aadp, src, sizeof (zil_chain_t) - sizeof (zio_eck_t)); aadp += sizeof (zil_chain_t) - sizeof (zio_eck_t); aad_len += sizeof (zil_chain_t) - sizeof (zio_eck_t); + /* loop over records again, filling in iovecs */ + nr_iovecs = 0; slrp = src + sizeof (zil_chain_t); dlrp = dst + sizeof (zil_chain_t); - /* - * Loop over records again, filling in iovecs. - */ - - /* The first iovec will contain the authbuf. */ - vec = 1; - for (; slrp < blkend; slrp += lr_len, dlrp += lr_len) { lr = (lr_t *)slrp; @@ -1337,6 +1220,9 @@ zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf, aadp += sizeof (lr_t); aad_len += sizeof (lr_t); + ASSERT3P(src_iovecs, !=, NULL); + ASSERT3P(dst_iovecs, !=, NULL); + /* * If this is a TX_WRITE record we want to encrypt everything * except the bp if exists. If the bp does exist we want to @@ -1345,63 +1231,64 @@ zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf, if (txtype == TX_WRITE) { const size_t o = offsetof(lr_write_t, lr_blkptr); crypt_len = o - sizeof (lr_t); - dst_iovecs[vec].iov_base = (char *)dlrp + sizeof (lr_t); - dst_iovecs[vec].iov_len = crypt_len; + src_iovecs[idx].iov_base = slrp + sizeof (lr_t); + src_iovecs[idx].iov_len = crypt_len; + dst_iovecs[idx].iov_base = dlrp + sizeof (lr_t); + dst_iovecs[idx].iov_len = crypt_len; + idx++; /* copy the bp now since it will not be encrypted */ memcpy(dlrp + o, slrp + o, sizeof (blkptr_t)); memcpy(aadp, slrp + o, sizeof (blkptr_t)); aadp += sizeof (blkptr_t); aad_len += sizeof (blkptr_t); - vec++; + nr_iovecs++; total_len += crypt_len; if (lr_len != sizeof (lr_write_t)) { crypt_len = lr_len - sizeof (lr_write_t); - dst_iovecs[vec].iov_base = (char *) + src_iovecs[idx].iov_base = + slrp + sizeof (lr_write_t); + src_iovecs[idx].iov_len = crypt_len; + dst_iovecs[idx].iov_base = dlrp + sizeof (lr_write_t); - dst_iovecs[vec].iov_len = crypt_len; - vec++; + dst_iovecs[idx].iov_len = crypt_len; + idx++; + nr_iovecs++; total_len += crypt_len; } } else if (txtype == TX_CLONE_RANGE) { const size_t o = offsetof(lr_clone_range_t, lr_nbps); crypt_len = o - sizeof (lr_t); - dst_iovecs[vec].iov_base = (char *)dlrp + sizeof (lr_t); - dst_iovecs[vec].iov_len = crypt_len; + src_iovecs[idx].iov_base = slrp + sizeof (lr_t); + src_iovecs[idx].iov_len = crypt_len; + dst_iovecs[idx].iov_base = dlrp + sizeof (lr_t); + dst_iovecs[idx].iov_len = crypt_len; + idx++; /* copy the bps now since they will not be encrypted */ memcpy(dlrp + o, slrp + o, lr_len - o); memcpy(aadp, slrp + o, lr_len - o); aadp += lr_len - o; aad_len += lr_len - o; - vec++; + nr_iovecs++; total_len += crypt_len; } else { crypt_len = lr_len - sizeof (lr_t); - dst_iovecs[vec].iov_base = (char *)dlrp + sizeof (lr_t); - dst_iovecs[vec].iov_len = crypt_len; - vec++; + src_iovecs[idx].iov_base = slrp + sizeof (lr_t); + src_iovecs[idx].iov_len = crypt_len; + dst_iovecs[idx].iov_base = dlrp + sizeof (lr_t); + dst_iovecs[idx].iov_len = crypt_len; + idx++; + nr_iovecs++; total_len += crypt_len; } } - /* The last iovec will contain the MAC. */ - ASSERT3U(vec, ==, nr_iovecs - 1); - - /* AAD */ - dst_iovecs[0].iov_base = aadbuf; - dst_iovecs[0].iov_len = aad_len; - /* MAC */ - dst_iovecs[vec].iov_base = 0; - dst_iovecs[vec].iov_len = 0; - - *no_crypt = (vec == 1); + *no_crypt = (nr_iovecs == 0); *enc_len = total_len; *authbuf = aadbuf; *auth_len = aad_len; - GET_UIO_STRUCT(out_uio)->uio_iov = dst_iovecs; - zfs_uio_iovcnt(out_uio) = nr_iovecs; return (0); } @@ -1412,16 +1299,16 @@ zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf, static int zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version, uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, - zfs_uio_t *puio, zfs_uio_t *out_uio, uint_t *enc_len, uint8_t **authbuf, + zfs_uio_t *puio, zfs_uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len, boolean_t *no_crypt) { - uint8_t *aadbuf = zio_buf_alloc(datalen); + uint_t crypt_len; + uint_t aad_len = 0, nr_iovecs = 0, total_len = 0; + uint_t i, j, max_dnp = datalen >> DNODE_SHIFT; uint8_t *src, *dst, *aadp; + int idx; + iovec_t *src_iovecs, *dst_iovecs; dnode_phys_t *dnp, *adnp, *sdnp, *ddnp; - iovec_t *dst_iovecs; - uint_t nr_iovecs, crypt_len, vec; - uint_t aad_len = 0, total_len = 0; - uint_t i, j, max_dnp = datalen >> DNODE_SHIFT; if (encrypt) { src = plainbuf; @@ -1430,20 +1317,14 @@ zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version, src = cipherbuf; dst = plainbuf; } - memcpy(dst, src, datalen); sdnp = (dnode_phys_t *)src; ddnp = (dnode_phys_t *)dst; - aadp = aadbuf; /* * Count the number of iovecs we will need to do the encryption by * counting the number of bonus buffers that need to be encrypted. */ - - /* We need at least two iovecs -- one for the AAD, one for the MAC. */ - nr_iovecs = 2; - for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { /* * This block may still be byteswapped. However, all of the @@ -1458,17 +1339,28 @@ zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version, } } - dst_iovecs = kmem_alloc(nr_iovecs * sizeof (iovec_t), KM_SLEEP); + int err = zio_crypt_uios_init_os(puio, cuio, nr_iovecs, &idx); + if (err != 0) + return (err); + + if (encrypt) { + src_iovecs = zfs_uio_iov(puio); + dst_iovecs = zfs_uio_iov(cuio); + } else { + src_iovecs = zfs_uio_iov(cuio); + dst_iovecs = zfs_uio_iov(puio); + } + + nr_iovecs = 0; + + uint8_t *aadbuf = zio_buf_alloc(datalen); + aadp = aadbuf; /* * Iterate through the dnodes again, this time filling in the uios * we allocated earlier. We also concatenate any data we want to * authenticate onto aadbuf. */ - - /* The first iovec will contain the authbuf. */ - vec = 1; - for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { dnp = &sdnp[i]; @@ -1523,10 +1415,15 @@ zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version, if (dnp->dn_type != DMU_OT_NONE && DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) && dnp->dn_bonuslen != 0) { - dst_iovecs[vec].iov_base = DN_BONUS(&ddnp[i]); - dst_iovecs[vec].iov_len = crypt_len; + ASSERT3P(src_iovecs, !=, NULL); + ASSERT3P(dst_iovecs, !=, NULL); + src_iovecs[idx].iov_base = DN_BONUS(dnp); + src_iovecs[idx].iov_len = crypt_len; + dst_iovecs[idx].iov_base = DN_BONUS(&ddnp[i]); + dst_iovecs[idx].iov_len = crypt_len; + idx++; - vec++; + nr_iovecs++; total_len += crypt_len; } else { memcpy(DN_BONUS(&ddnp[i]), DN_BONUS(dnp), crypt_len); @@ -1536,72 +1433,34 @@ zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version, } } - /* The last iovec will contain the MAC. */ - ASSERT3U(vec, ==, nr_iovecs - 1); - - /* AAD */ - dst_iovecs[0].iov_base = aadbuf; - dst_iovecs[0].iov_len = aad_len; - /* MAC */ - dst_iovecs[vec].iov_base = 0; - dst_iovecs[vec].iov_len = 0; - - *no_crypt = (vec == 1); + *no_crypt = (nr_iovecs == 0); *enc_len = total_len; *authbuf = aadbuf; *auth_len = aad_len; - GET_UIO_STRUCT(out_uio)->uio_iov = dst_iovecs; - zfs_uio_iovcnt(out_uio) = nr_iovecs; return (0); } static int zio_crypt_init_uios_normal(boolean_t encrypt, uint8_t *plainbuf, - uint8_t *cipherbuf, uint_t datalen, zfs_uio_t *puio, zfs_uio_t *out_uio, + uint8_t *cipherbuf, uint_t datalen, zfs_uio_t *puio, zfs_uio_t *cuio, uint_t *enc_len) { - (void) puio; - int ret; - uint_t nr_plain = 1, nr_cipher = 2; - iovec_t *plain_iovecs = NULL, *cipher_iovecs = NULL; - void *src, *dst; - - cipher_iovecs = kmem_zalloc(nr_cipher * sizeof (iovec_t), - KM_SLEEP); - if (!cipher_iovecs) { - ret = SET_ERROR(ENOMEM); - goto error; - } + (void) encrypt; - if (encrypt) { - src = plainbuf; - dst = cipherbuf; - } else { - src = cipherbuf; - dst = plainbuf; - } - memcpy(dst, src, datalen); - cipher_iovecs[0].iov_base = dst; - cipher_iovecs[0].iov_len = datalen; + int idx; + int err = zio_crypt_uios_init_os(puio, cuio, 1, &idx); + if (err != 0) + return (err); + + zfs_uio_iovbase(puio, idx) = plainbuf; + zfs_uio_iovlen(puio, idx) = datalen; + zfs_uio_iovbase(cuio, idx) = cipherbuf; + zfs_uio_iovlen(cuio, idx) = datalen; *enc_len = datalen; - GET_UIO_STRUCT(out_uio)->uio_iov = cipher_iovecs; - zfs_uio_iovcnt(out_uio) = nr_cipher; return (0); - -error: - if (plain_iovecs != NULL) - kmem_free(plain_iovecs, nr_plain * sizeof (iovec_t)); - if (cipher_iovecs != NULL) - kmem_free(cipher_iovecs, nr_cipher * sizeof (iovec_t)); - - *enc_len = 0; - GET_UIO_STRUCT(out_uio)->uio_iov = NULL; - zfs_uio_iovcnt(out_uio) = 0; - - return (ret); } /* @@ -1615,28 +1474,27 @@ zio_crypt_init_uios_normal(boolean_t encrypt, uint8_t *plainbuf, static int zio_crypt_init_uios(boolean_t encrypt, uint64_t version, dmu_object_type_t ot, uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, - uint8_t *mac, zfs_uio_t *puio, zfs_uio_t *cuio, uint_t *enc_len, + zfs_uio_t *puio, zfs_uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len, boolean_t *no_crypt) { - int ret; - iovec_t *mac_iov; + int err; ASSERT(DMU_OT_IS_ENCRYPTED(ot) || ot == DMU_OT_NONE); /* route to handler */ switch (ot) { case DMU_OT_INTENT_LOG: - ret = zio_crypt_init_uios_zil(encrypt, plainbuf, cipherbuf, + err = zio_crypt_init_uios_zil(encrypt, plainbuf, cipherbuf, datalen, byteswap, puio, cuio, enc_len, authbuf, auth_len, no_crypt); break; case DMU_OT_DNODE: - ret = zio_crypt_init_uios_dnode(encrypt, version, plainbuf, + err = zio_crypt_init_uios_dnode(encrypt, version, plainbuf, cipherbuf, datalen, byteswap, puio, cuio, enc_len, authbuf, auth_len, no_crypt); break; default: - ret = zio_crypt_init_uios_normal(encrypt, plainbuf, cipherbuf, + err = zio_crypt_init_uios_normal(encrypt, plainbuf, cipherbuf, datalen, puio, cuio, enc_len); *authbuf = NULL; *auth_len = 0; @@ -1644,22 +1502,10 @@ zio_crypt_init_uios(boolean_t encrypt, uint64_t version, dmu_object_type_t ot, break; } - if (ret != 0) - goto error; - - /* populate the uios */ - zfs_uio_segflg(cuio) = UIO_SYSSPACE; - - mac_iov = - ((iovec_t *)&(GET_UIO_STRUCT(cuio)-> - uio_iov[zfs_uio_iovcnt(cuio) - 1])); - mac_iov->iov_base = (void *)mac; - mac_iov->iov_len = ZIO_DATA_MAC_LEN; + if (err != 0) + return (err); return (0); - -error: - return (ret); } /* @@ -1673,40 +1519,16 @@ zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, { int ret; boolean_t locked = B_FALSE; - uint64_t crypt = key->zk_crypt; - uint_t keydata_len = zio_crypt_table[crypt].ci_keylen; + const zio_crypt_info_t *ci = &zio_crypt_table[key->zk_crypt]; uint_t enc_len, auth_len; zfs_uio_t puio, cuio; - struct uio puio_s, cuio_s; uint8_t enc_keydata[MASTER_KEY_MAX_LEN]; crypto_key_t tmp_ckey, *ckey = NULL; - freebsd_crypt_session_t *tmpl = NULL; + zio_crypt_session_t *sess; uint8_t *authbuf = NULL; - memset(&puio_s, 0, sizeof (puio_s)); - memset(&cuio_s, 0, sizeof (cuio_s)); - zfs_uio_init(&puio, &puio_s); - zfs_uio_init(&cuio, &cuio_s); - -#ifdef FCRYPTO_DEBUG - printf("%s(%s, %p, %p, %d, %p, %p, %u, %s, %p, %p, %p)\n", - __FUNCTION__, - encrypt ? "encrypt" : "decrypt", - key, salt, ot, iv, mac, datalen, - byteswap ? "byteswap" : "native_endian", plainbuf, - cipherbuf, no_crypt); - - printf("\tkey = {"); - for (int i = 0; i < key->zk_current_key.ck_length/8; i++) - printf("%02x ", ((uint8_t *)key->zk_current_key.ck_data)[i]); - printf("}\n"); -#endif - /* create uios for encryption */ - ret = zio_crypt_init_uios(encrypt, key->zk_version, ot, plainbuf, - cipherbuf, datalen, byteswap, mac, &puio, &cuio, &enc_len, - &authbuf, &auth_len, no_crypt); - if (ret != 0) - return (ret); + memset(&puio, 0, sizeof (puio)); + memset(&cuio, 0, sizeof (cuio)); /* * If the needed key is the current one, just use it. Otherwise we @@ -1719,27 +1541,71 @@ zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, if (memcmp(salt, key->zk_salt, ZIO_DATA_SALT_LEN) == 0) { ckey = &key->zk_current_key; - tmpl = &key->zk_session; + sess = &key->zk_current_sess; } else { rw_exit(&key->zk_salt_lock); locked = B_FALSE; - ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, - salt, ZIO_DATA_SALT_LEN, enc_keydata, keydata_len); + ret = hkdf_sha512(key->zk_master_keydata, ci->ci_keylen, NULL, + 0, salt, ZIO_DATA_SALT_LEN, enc_keydata, ci->ci_keylen); if (ret != 0) goto error; + tmp_ckey.ck_data = enc_keydata; - tmp_ckey.ck_length = CRYPTO_BYTES2BITS(keydata_len); + tmp_ckey.ck_length = CRYPTO_BYTES2BITS(ci->ci_keylen); ckey = &tmp_ckey; - tmpl = NULL; + sess = NULL; } - /* perform the encryption / decryption */ - ret = zio_do_crypt_uio_opencrypto(encrypt, tmpl, key->zk_crypt, - ckey, iv, enc_len, &cuio, auth_len); + /* + * Attempt to use QAT acceleration if we can. We currently don't + * do this for metadnode and ZIL blocks, since they have a much + * more involved buffer layout and the qat_crypt() function only + * works in-place. + */ + if (qat_crypt_use_accel(datalen) && + ot != DMU_OT_INTENT_LOG && ot != DMU_OT_DNODE) { + uint8_t *srcbuf, *dstbuf; + + if (encrypt) { + srcbuf = plainbuf; + dstbuf = cipherbuf; + } else { + srcbuf = cipherbuf; + dstbuf = plainbuf; + } + + ret = qat_crypt((encrypt) ? QAT_ENCRYPT : QAT_DECRYPT, srcbuf, + dstbuf, NULL, 0, iv, mac, ckey, key->zk_crypt, datalen); + if (ret == CPA_STATUS_SUCCESS) { + if (locked) { + rw_exit(&key->zk_salt_lock); + locked = B_FALSE; + } + + return (0); + } + /* If the hardware implementation fails fall back to software */ + } + + /* create uios for encryption */ + ret = zio_crypt_init_uios(encrypt, key->zk_version, ot, plainbuf, + cipherbuf, datalen, byteswap, &puio, &cuio, &enc_len, + &authbuf, &auth_len, no_crypt); if (ret != 0) goto error; + + /* perform the encryption / decryption in software */ + if (encrypt) + ret = zio_encrypt_os(ci, ckey, sess, &puio, &cuio, enc_len, + iv, authbuf, auth_len, mac); + else + ret = zio_decrypt_os(ci, ckey, sess, &cuio, &puio, enc_len, + iv, authbuf, auth_len, mac); + if (ret != 0) + goto error; + if (locked) { rw_exit(&key->zk_salt_lock); } @@ -1747,9 +1613,9 @@ zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, if (authbuf != NULL) zio_buf_free(authbuf, datalen); if (ckey == &tmp_ckey) - memset(enc_keydata, 0, keydata_len); - zio_crypt_destroy_uio(&puio); - zio_crypt_destroy_uio(&cuio); + memset(enc_keydata, 0, ci->ci_keylen); + + zio_crypt_uios_fini_os(&puio, &cuio); return (0); @@ -1759,9 +1625,8 @@ zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, if (authbuf != NULL) zio_buf_free(authbuf, datalen); if (ckey == &tmp_ckey) - memset(enc_keydata, 0, keydata_len); - zio_crypt_destroy_uio(&puio); - zio_crypt_destroy_uio(&cuio); + memset(enc_keydata, 0, ci->ci_keylen); + zio_crypt_uios_fini_os(&puio, &cuio); return (SET_ERROR(ret)); } @@ -1811,3 +1676,7 @@ zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot, return (SET_ERROR(ret)); } + +ZFS_MODULE_PARAM(zfs, zfs_key_, max_salt_uses, U64, ZMOD_RW, + "Max number of times a salt value can be used for generating " + "encryption keys before it is rotated"); diff --git a/module/zfs/zio_crypt_os_icp.c b/module/zfs/zio_crypt_os_icp.c new file mode 100644 index 000000000000..a9170007c5d5 --- /dev/null +++ b/module/zfs/zio_crypt_os_icp.c @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + */ + +/* + * Copyright (c) 2017, Datto, Inc. All rights reserved. + * Copyright (c) 2026, TrueNAS. + */ + +#include + +void +zio_crypt_key_close_os(zio_crypt_key_t *key) +{ + /* free crypto templates */ + crypto_destroy_ctx_template(key->zk_current_sess.zs_tmpl); + crypto_destroy_ctx_template(key->zk_hmac_sess.zs_tmpl); +} + +int +zio_crypt_key_open_os(zio_crypt_key_t *key, const zio_crypt_info_t *ci) +{ + crypto_mechanism_t mech = {0}; + int ret; + + /* + * Initialize the crypto templates. It's ok if this fails because + * this is just an optimization. + */ + mech.cm_type = crypto_mech2id(ci->ci_mechname); + ret = crypto_create_ctx_template(&mech, &key->zk_current_key, + &key->zk_current_sess.zs_tmpl); + if (ret != CRYPTO_SUCCESS) + key->zk_current_sess.zs_tmpl = NULL; + + mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); + ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key, + &key->zk_hmac_sess.zs_tmpl); + if (ret != CRYPTO_SUCCESS) + key->zk_hmac_sess.zs_tmpl = NULL; + + return (0); +} + +int +zio_crypt_key_reopen_os(zio_crypt_key_t *key, const zio_crypt_info_t *ci) +{ + int ret; + crypto_mechanism_t mech = {0}; + + crypto_destroy_ctx_template(key->zk_current_sess.zs_tmpl); + + mech.cm_type = crypto_mech2id(ci->ci_mechname); + ret = crypto_create_ctx_template(&mech, &key->zk_current_key, + &key->zk_current_sess.zs_tmpl); + if (ret != CRYPTO_SUCCESS) + key->zk_current_sess.zs_tmpl = NULL; + return (0); +} + +/* + * Initialise a pair of uios with the requested number of data iovecs. An + * additional iovec will be allocated for at the end for the ICP to use for the + * MAC buffer. + */ +int +zio_crypt_uios_init_os(zfs_uio_t *u1, zfs_uio_t *u2, int iovcnt, int *idx) +{ + memset(u1, 0, sizeof (zfs_uio_t)); + memset(u2, 0, sizeof (zfs_uio_t)); + + /* One extra for the MAC. */ + iovcnt++; + + zfs_uio_iov(u1) = kmem_zalloc(iovcnt * sizeof (iovec_t), KM_SLEEP); + zfs_uio_iov(u2) = kmem_zalloc(iovcnt * sizeof (iovec_t), KM_SLEEP); + + zfs_uio_iovcnt(u1) = zfs_uio_iovcnt(u2) = iovcnt; + zfs_uio_segflg(u1) = zfs_uio_segflg(u2) = UIO_SYSSPACE; + + *idx = 0; + + return (0); +} + +void +zio_crypt_uios_fini_os(zfs_uio_t *u1, zfs_uio_t *u2) { + ASSERT3U(zfs_uio_iovcnt(u1), ==, zfs_uio_iovcnt(u2)); + + kmem_free(zfs_uio_iov(u1), zfs_uio_iovcnt(u1) * sizeof (iovec_t)); + kmem_free(zfs_uio_iov(u2), zfs_uio_iovcnt(u2) * sizeof (iovec_t)); +} + +static int +zio_encrypt_decrypt_os_common(boolean_t do_encrypt, const zio_crypt_info_t *ci, + crypto_key_t *key, zio_crypt_session_t *sess, + zfs_uio_t *src, zfs_uio_t *dst, size_t datalen, + const uint8_t iv[ZIO_DATA_IV_LEN], const uint8_t *ad, size_t adlen, + uint8_t mac[ZIO_DATA_MAC_LEN]) +{ + ASSERT3U(zfs_uio_iovcnt(src), ==, zfs_uio_iovcnt(dst)); + + /* populate the source and dest structs */ + crypto_data_t src_cd = { + .cd_format = CRYPTO_DATA_UIO, + .cd_uio = src, + .cd_offset = 0, + }; + crypto_data_t dst_cd = { + .cd_format = CRYPTO_DATA_UIO, + .cd_uio = dst, + .cd_offset = 0, + }; + + if (do_encrypt) { + /* + * When encrypting, the MAC will be stored at the end of the + * encrypted data, so add the MAC buffer to the end of the dest + * UIO and extend the data length to accomodate it. + */ + zfs_uio_iovbase(dst, zfs_uio_iovcnt(dst)-1) = mac; + zfs_uio_iovlen(dst, zfs_uio_iovcnt(dst)-1) = ZIO_DATA_MAC_LEN; + src_cd.cd_length = datalen; + dst_cd.cd_length = datalen + ZIO_DATA_MAC_LEN; + } else { + /* + * When decrypting, the MAC is presented at the end of the + * plaintext data, so add the MAC buffer to the end of the + * source UIO. + * + * Strangely, the ICP requires that the destination/plaintext + * must include the MAC length when decrypting, even though it + * will never write anything and does not need to have the + * extra space allocated. So we extend both source and data + * length to cover it. + */ + zfs_uio_iovbase(src, zfs_uio_iovcnt(src)-1) = mac; + zfs_uio_iovlen(src, zfs_uio_iovcnt(src)-1) = ZIO_DATA_MAC_LEN; + src_cd.cd_length = dst_cd.cd_length = + datalen + ZIO_DATA_MAC_LEN; + } + + /* setup encryption mechanism */ + crypto_mechanism_t mech = {0}; + mech.cm_type = crypto_mech2id(ci->ci_mechname); + + /* setup encryption params */ + union { + CK_AES_CCM_PARAMS ccm; + CK_AES_GCM_PARAMS gcm; + } params; + + if (ci->ci_crypt_type == ZC_TYPE_CCM) { + CK_AES_CCM_PARAMS *ccm = ¶ms.ccm; + ccm->nonce = (uchar_t *)iv; + ccm->ulNonceSize = ZIO_DATA_IV_LEN; + ccm->authData = (uchar_t *)ad; + ccm->ulAuthDataSize = adlen; + ccm->ulDataSize = src_cd.cd_length; + ccm->ulMACSize = ZIO_DATA_MAC_LEN; + mech.cm_param = (caddr_t)ccm; + mech.cm_param_len = sizeof (*ccm); + } else { + CK_AES_GCM_PARAMS *gcm = ¶ms.gcm; + gcm->pIv = (uchar_t *)iv; + gcm->ulIvLen = ZIO_DATA_IV_LEN; + gcm->ulIvBits = CRYPTO_BYTES2BITS(ZIO_DATA_IV_LEN); + gcm->pAAD = (uchar_t *)ad; + gcm->ulAADLen = adlen; + gcm->ulTagBits = CRYPTO_BYTES2BITS(ZIO_DATA_MAC_LEN); + mech.cm_param = (caddr_t)gcm; + mech.cm_param_len = sizeof (*gcm); + } + + /* perform the actual encryption */ + crypto_ctx_template_t *tmpl = sess != NULL ? sess->zs_tmpl : NULL; + int err = 0; + if (do_encrypt) { + err = crypto_encrypt(&mech, &src_cd, key, tmpl, &dst_cd); + if (err != CRYPTO_SUCCESS) + err = SET_ERROR(EIO); + } else { + err = crypto_decrypt(&mech, &src_cd, key, tmpl, &dst_cd); + if (err != CRYPTO_SUCCESS) { + ASSERT3U(err, ==, CRYPTO_INVALID_MAC); + err = SET_ERROR(ECKSUM); + } + } + + return (err); +} + +int +zio_encrypt_os(const zio_crypt_info_t *ci, + crypto_key_t *key, zio_crypt_session_t *sess, + zfs_uio_t *plaintext, zfs_uio_t *ciphertext, size_t datalen, + const uint8_t iv[ZIO_DATA_IV_LEN], const uint8_t *ad, size_t adlen, + uint8_t mac[ZIO_DATA_MAC_LEN]) +{ + return (zio_encrypt_decrypt_os_common(B_TRUE, ci, key, sess, + plaintext, ciphertext, datalen, iv, ad, adlen, mac)); +} + +int +zio_decrypt_os(const zio_crypt_info_t *ci, + crypto_key_t *key, zio_crypt_session_t *sess, + zfs_uio_t *ciphertext, zfs_uio_t *plaintext, size_t datalen, + const uint8_t iv[ZIO_DATA_IV_LEN], const uint8_t *ad, size_t adlen, + uint8_t mac[ZIO_DATA_MAC_LEN]) +{ + return (zio_encrypt_decrypt_os_common(B_FALSE, ci, key, sess, + ciphertext, plaintext, datalen, iv, ad, adlen, mac)); +} + +int +zio_crypt_hmac_os(zio_crypt_key_t *key, const uint8_t *data, size_t datalen, + uint8_t digest[SHA512_HMAC_LEN]) +{ + /* initialize sha512-hmac mechanism */ + crypto_mechanism_t mech = {0}; + mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); + + /* initialize the crypto data */ + crypto_data_t in = { + .cd_format = CRYPTO_DATA_RAW, + .cd_offset = 0, + .cd_length = datalen, + .cd_raw.iov_base = (void *)data, + .cd_raw.iov_len = datalen, + }; + crypto_data_t out = { + .cd_format = CRYPTO_DATA_RAW, + .cd_offset = 0, + .cd_length = SHA512_HMAC_LEN, + .cd_raw.iov_base = (void *)digest, + .cd_raw.iov_len = SHA512_HMAC_LEN, + }; + + /* generate the hmac */ + if (crypto_mac(&mech, &in, &key->zk_hmac_key, + key->zk_hmac_sess.zs_tmpl, &out) != CRYPTO_SUCCESS) + return (SET_ERROR(EIO)); + + return (0); +} + +int +zio_crypt_hmac_init_os(zio_crypt_hmac_t *hmac, zio_crypt_key_t *key) +{ + /* initialize HMAC mechanism */ + crypto_mechanism_t mech = {0}; + mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); + + if (crypto_mac_init(&mech, &key->zk_hmac_key, NULL, + &hmac->zh_ctx) != CRYPTO_SUCCESS) + return (SET_ERROR(EIO)); + + return (0); +} + +int +zio_crypt_hmac_update_os(zio_crypt_hmac_t *hmac, const uint8_t *data, + size_t datalen) +{ + crypto_data_t cd = { + .cd_format = CRYPTO_DATA_RAW, + .cd_offset = 0, + .cd_length = datalen, + .cd_raw.iov_base = (char *)data, + .cd_raw.iov_len = datalen, + }; + + if (crypto_mac_update(hmac->zh_ctx, &cd) != CRYPTO_SUCCESS) + return (SET_ERROR(EIO)); + + return (0); +} + +int +zio_crypt_hmac_final_os(zio_crypt_hmac_t *hmac, uint8_t digest[SHA512_HMAC_LEN]) +{ + crypto_data_t cd = { + .cd_format = CRYPTO_DATA_RAW, + .cd_offset = 0, + .cd_length = SHA512_HMAC_LEN, + .cd_raw.iov_base = (char *)digest, + .cd_raw.iov_len = SHA512_HMAC_LEN, + }; + + if (crypto_mac_final(hmac->zh_ctx, &cd) != CRYPTO_SUCCESS) + return (SET_ERROR(EIO)); + + return (0); +}