From 34f3d082bb7744a347b7ccdb09c6209191713aaf Mon Sep 17 00:00:00 2001 From: Xiaoxu Zeng Date: Thu, 6 Jul 2023 15:29:56 +0800 Subject: [PATCH 1/3] drivers: crypto: hisilicon: implement QM driver The Hisilicon QM is a Queue Management module. In order to unify the interface between accelerator and software, a unified queue management module QM is used to interact with software. Each accelerator module integrates a QM. Software issues tasks to the SQ (Submmision Queue),and the QM obtains the address of the SQE (Submmision Queue Element). The BD (Buffer Description, same as SQE) information is sent to the accelerator. After the task processing is complete, the accelerator applies for a write-back address from the QM to write back the SQ. Signed-off-by: Xiaoxu Zeng --- core/arch/arm/plat-d06/conf.mk | 2 + core/drivers/crypto/hisilicon/hisi_qm.c | 786 ++++++++++++++++++ .../crypto/hisilicon/include/hisi_qm.h | 276 ++++++ core/drivers/crypto/hisilicon/sub.mk | 3 + core/drivers/crypto/sub.mk | 2 + 5 files changed, 1069 insertions(+) create mode 100644 core/drivers/crypto/hisilicon/hisi_qm.c create mode 100644 core/drivers/crypto/hisilicon/include/hisi_qm.h create mode 100644 core/drivers/crypto/hisilicon/sub.mk diff --git a/core/arch/arm/plat-d06/conf.mk b/core/arch/arm/plat-d06/conf.mk index 98a8fa01bbe..b0830863dc7 100644 --- a/core/arch/arm/plat-d06/conf.mk +++ b/core/arch/arm/plat-d06/conf.mk @@ -8,7 +8,9 @@ CFG_WITH_PAGER ?= n CFG_WITH_SOFTWARE_PRNG ?= y CFG_WITH_STATS ?= y CFG_TEE_CORE_EMBED_INTERNAL_TESTS ?= y +CFG_HISILICON_CRYPTO_DRIVER ?= y +$(call force,CFG_GIC,y) $(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) $(call force,CFG_WITH_ARM_TRUSTED_FW,y) $(call force,CFG_ARM64_core,y) diff --git a/core/drivers/crypto/hisilicon/hisi_qm.c b/core/drivers/crypto/hisilicon/hisi_qm.c new file mode 100644 index 00000000000..6c1e5960e0c --- /dev/null +++ b/core/drivers/crypto/hisilicon/hisi_qm.c @@ -0,0 +1,786 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 HiSilicon Limited. + * Kunpeng hardware accelerator queue management module. + */ +#include "hisi_qm.h" + +#define QM_FVT_CFG_RDY_BIT 0x1 +/* doorbell */ +#define QM_DOORBELL_SQ_CQ_BASE 0x1000 +#define QM_DB_CMD_SHIFT 12 +#define QM_DB_RAND_DATA_SHIFT 16 +#define QM_DB_INDEX_SHIFT 32 +#define QM_DB_PRIORITY_SHIFT 48 +#define QM_DB_RAND_DATA 0x5a +#define QM_DOORBELL_CMD_SQ 0 +#define QM_DOORBELL_CMD_CQ 1 +/* mailbox */ +#define QM_MAILBOX_BASE 0x300 +#define QM_MAILBOX_DATA_ADDR_L 0x304 +#define QM_MAILBOX_DATA_ADDR_H 0x308 +#define QM_MB_BUSY_SHIFT 13 +#define QM_MB_BUSY_BIT BIT32(QM_MB_BUSY_SHIFT) +#define QM_MB_OP_SHIFT 14 +#define QM_MB_OP_WR 0 +#define QM_MB_OP_RD 1 +/* XQC_VFT */ +#define QM_VFT_CFG_OP_ENABLE 0x100054 +#define QM_VFT_CFG_OP_WR 0x100058 +#define QM_VFT_CFG_TYPE 0x10005c +#define QM_VFT_CFG_ADDRESS 0x100060 +#define QM_VFT_CFG_DATA_L 0x100064 +#define QM_VFT_CFG_DATA_H 0x100068 +#define QM_VFT_CFG_RDY 0x10006c +#define QM_SQC_VFT 0 +#define QM_CQC_VFT 1 +#define QM_SQC_VFT_START_SQN_SHIFT 28 +#define QM_SQC_VFT_VALID BIT64(44) +#define QM_SQC_VFT_SQ_NUM_SHIFT 45 +#define QM_CQC_VFT_VALID BIT(28) +#define QM_VFT_WRITE 0 +#define QM_VFT_READ 1 +#define QM_SQC_VFT_BASE_MASK 0x3ff +#define QM_SQC_VFT_NUM_MASK 0x3ff +/* QM INIT */ +#define QM_MEM_START_INIT 0x100040 +#define QM_MEM_INIT_DONE 0x100044 +#define QM_VF_AEQ_INT_MASK 0x4 +#define QM_VF_AEQ_INT_MASK_EN 0x1 +#define QM_VF_EQ_INT_MASK 0xc +#define QM_VF_EQ_INT_MASK_EN 0x1 +#define QM_ARUSER_M_CFG_1 0x100088 +#define QM_ARUSER_M_CFG_ENABLE 0x100090 +#define QM_AWUSER_M_CFG_1 0x100098 +#define QM_AWUSER_M_CFG_ENABLE 0x1000a0 +#define QM_AXUSER_CFG 0x40001070 +#define AXUSER_M_CFG_ENABLE 0x7ffffc +#define QM_AXI_M_CFG 0x1000ac +#define AXI_M_CFG 0xffff +#define QM_PEH_AXUSER_CFG 0x1000cc +#define PEH_AXUSER_CFG 0x400801 +#define QM_CACHE_CTL 0x100050 +#define QM_CACHE_CFG 0x4893 +#define QM_CACHE_WB_START 0x100204 +#define QM_CACHE_WB_DONE 0x100208 +/* XQC shift */ +#define QM_SQ_SQE_SIZE_SHIFT 12 +#define QM_SQ_ORDER_SHIFT 4 +#define QM_SQ_TYPE_SHIFT 8 +#define QM_CQE_SIZE 4 +#define QM_CQ_CQE_SIZE_SHIFT 12 +/* CQE */ +#define QM_CQE_PHASE(cqe) (((cqe)->w7) & QM_FVT_CFG_RDY_BIT) + +enum qm_mailbox_common_cmd { + QM_MB_CMD_SQC = 0x0, + QM_MB_CMD_CQC, + QM_MB_CMD_EQC, + QM_MB_CMD_AEQC, + QM_MB_CMD_SQC_BT, + QM_MB_CMD_CQC_BT, + QM_MB_CMD_SQC_VFT, +}; + +enum qm_mailbox_cmd_v3 { + QM_MB_CM_CLOSE_QM = 0x7, + QM_MB_CMD_CLOSE_QP, + QM_MB_CMD_FLUSH_QM, + QM_MB_CMD_FLUSH_QP, + QM_MB_CMD_SRC = 0xc, + QM_MB_CMD_DST, + QM_MB_CMD_STOP_QM, +}; + +struct qm_mailbox { + uint16_t w0; + uint16_t queue; + uint32_t base_l; + uint32_t base_h; + uint32_t token; +}; + +struct qm_dfx_registers { + const char *reg_name; + uint32_t reg_offset; +}; + +static const struct qm_dfx_registers qm_dfx_regs[] = { + { .reg_name = "QM_ECC_1BIT_CNT ", .reg_offset = 0x104000 }, + { .reg_name = "QM_ECC_MBIT_CNT ", .reg_offset = 0x104008 }, + { .reg_name = "QM_DFX_MB_CNT ", .reg_offset = 0x104018 }, + { .reg_name = "QM_DFX_DB_CNT ", .reg_offset = 0x104028 }, + { .reg_name = "QM_DFX_SQE_CNT ", .reg_offset = 0x104038 }, + { .reg_name = "QM_DFX_CQE_CNT ", .reg_offset = 0x104048 }, + { .reg_name = "QM_DFX_SEND_SQE_TO_ACC_CNT", .reg_offset = 0x104050 }, + { .reg_name = "QM_DFX_WB_SQE_FROM_ACC_CNT", .reg_offset = 0x104058 }, + { .reg_name = "QM_DFX_ACC_FINISH_CNT ", .reg_offset = 0x104060 }, + { .reg_name = "QM_DFX_CQE_ERR_CNT ", .reg_offset = 0x1040b4 }, + { .reg_name = NULL, 0 } +}; + +void hisi_qm_get_version(struct hisi_qm *qm) +{ + qm->version = io_read32(qm->io_base + HISI_QM_REVISON_ID_BASE) & + HISI_QM_REVISON_ID_MASK; +} + +static void qm_db(struct hisi_qm *qm, uint16_t qn, uint8_t cmd, uint16_t index, + uint8_t priority) +{ + uint64_t doorbell = 0; + + doorbell = qn | SHIFT_U64(cmd, QM_DB_CMD_SHIFT) | + SHIFT_U64(QM_DB_RAND_DATA, QM_DB_RAND_DATA_SHIFT) | + SHIFT_U64(index, QM_DB_INDEX_SHIFT) | + SHIFT_U64(priority, QM_DB_PRIORITY_SHIFT); + + io_write64(qm->io_base + QM_DOORBELL_SQ_CQ_BASE, doorbell); +} + +static enum hisi_drv_status qm_wait_mb_ready(struct hisi_qm *qm) +{ + uint32_t val = 0; + + /* return 0 mailbox ready, HISI_QM_DRVCRYPT_ETMOUT hardware timeout */ + return readl_relaxed_poll_timeout(qm->io_base + QM_MAILBOX_BASE, val, + !(val & QM_MB_BUSY_BIT), POLL_PERIOD, + POLL_TIMEOUT); +} + +static void qm_mb_write(struct hisi_qm *qm, void *src) +{ + vaddr_t dst = qm->io_base + QM_MAILBOX_BASE; + unsigned long tmp0 = 0; + unsigned long tmp1 = 0; + + /* 128bits should be written to hardware at one time */ + asm volatile ("ldp %0, %1, %3\n" + "stp %0, %1, %2\n" + : "=&r"(tmp0), "=&r"(tmp1), "+Q"(*((char *)dst)) + : "Q"(*((char *)src)) + : "memory"); + dsb(); +} + +static enum hisi_drv_status qm_mb(struct hisi_qm *qm, uint8_t cmd, + vaddr_t dma_addr, uint16_t qn, uint8_t op) +{ + struct qm_mailbox mb = { }; + + mb.w0 = cmd | SHIFT_U32(op, QM_MB_OP_SHIFT) | + BIT32(QM_MB_BUSY_SHIFT); + mb.queue = qn; + reg_pair_from_64(dma_addr, &mb.base_h, &mb.base_l); + mb.token = 0; + + if (qm_wait_mb_ready(qm)) { + EMSG("QM mailbox is busy"); + return HISI_QM_DRVCRYPT_EBUSY; + } + + qm_mb_write(qm, &mb); + + if (qm_wait_mb_ready(qm)) { + EMSG("QM mailbox operation timeout"); + return HISI_QM_DRVCRYPT_EBUSY; + } + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static void qm_cfg_vft_data(struct hisi_qm *qm, uint8_t vft_type, + uint32_t base, uint32_t number) +{ + uint32_t data_h = 0; + uint32_t data_l = 0; + uint64_t data = 0; + + switch (vft_type) { + case QM_SQC_VFT: + data = SHIFT_U64(base, QM_SQC_VFT_START_SQN_SHIFT) | + QM_SQC_VFT_VALID | + SHIFT_U64((number - 1), QM_SQC_VFT_SQ_NUM_SHIFT); + break; + case QM_CQC_VFT: + data = QM_CQC_VFT_VALID; + break; + default: + panic("Invalid vft type"); + } + + reg_pair_from_64(data, &data_h, &data_l); + io_write32(qm->io_base + QM_VFT_CFG_DATA_L, data_l); + io_write32(qm->io_base + QM_VFT_CFG_DATA_H, data_h); +} + +static enum hisi_drv_status qm_set_vft_common(struct hisi_qm *qm, + uint8_t vft_type, + uint32_t function, + uint32_t base, + uint32_t num) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + uint32_t val = 0; + + ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, + val & QM_FVT_CFG_RDY_BIT, POLL_PERIOD, + POLL_TIMEOUT); + if (ret) { + EMSG("QM VFT is not ready"); + return ret; + } + + io_write32(qm->io_base + QM_VFT_CFG_OP_WR, QM_VFT_WRITE); + io_write32(qm->io_base + QM_VFT_CFG_TYPE, vft_type); + io_write32(qm->io_base + QM_VFT_CFG_ADDRESS, function); + qm_cfg_vft_data(qm, vft_type, base, num); + io_write32(qm->io_base + QM_VFT_CFG_RDY, 0x0); + io_write32(qm->io_base + QM_VFT_CFG_OP_ENABLE, QM_FVT_CFG_RDY_BIT); + + return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, + val & QM_FVT_CFG_RDY_BIT, POLL_PERIOD, + POLL_TIMEOUT); +} + +static enum hisi_drv_status qm_set_xqc_vft(struct hisi_qm *qm, + uint32_t function, + uint32_t base, uint32_t num) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + int32_t i = 0; + + if (!num) { + EMSG("Invalid sq num"); + return HISI_QM_DRVCRYPT_EINVAL; + } + + for (i = QM_SQC_VFT; i <= QM_CQC_VFT; i++) { + ret = qm_set_vft_common(qm, i, function, base, num); + if (ret) { + EMSG("QM set type%d fail", i); + return ret; + } + } + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static enum hisi_drv_status qm_get_vft(struct hisi_qm *qm, uint32_t *base, + uint32_t *num) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + uint64_t sqc_vft = 0; + + ret = qm_mb(qm, QM_MB_CMD_SQC_VFT, 0, 0, QM_MB_OP_RD); + if (ret) + return ret; + + sqc_vft = io_read64(qm->io_base + QM_MAILBOX_DATA_ADDR_L); + *base = (sqc_vft >> QM_SQC_VFT_START_SQN_SHIFT) & QM_SQC_VFT_BASE_MASK; + *num = ((sqc_vft >> QM_SQC_VFT_SQ_NUM_SHIFT) & QM_SQC_VFT_NUM_MASK) + 1; + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static void qp_memory_uninit(struct hisi_qm *qm, uint32_t id) +{ + struct hisi_qp *qp = &qm->qp_array[id]; + + free(qp->sqe); + free(qp->cqe); +} + +static enum hisi_drv_status qp_memory_init(struct hisi_qm *qm, uint32_t id) +{ + size_t sq_size = qm->sqe_size * HISI_QM_Q_DEPTH; + size_t cq_size = sizeof(struct qm_cqe) * HISI_QM_Q_DEPTH; + struct hisi_qp *qp = &qm->qp_array[id]; + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + + qp->sqe = memalign(HISI_QM_ALIGN128, sq_size); + if (!qp->sqe) { + EMSG("Fail to malloc sq[%"PRIu32"]", id); + return HISI_QM_DRVCRYPT_ENOMEM; + } + qp->sqe_dma = virt_to_phys(qp->sqe); + qp->cqe = memalign(HISI_QM_ALIGN32, cq_size); + if (!qp->cqe) { + EMSG("Fail to malloc cq[%"PRIu32"]", id); + ret = HISI_QM_DRVCRYPT_ENOMEM; + goto free_sqe; + } + qp->cqe_dma = virt_to_phys(qp->cqe); + + qp->qp_id = id; + qp->qm = qm; + return HISI_QM_DRVCRYPT_NO_ERR; + +free_sqe: + free(qp->sqe); + return ret; +} + +static void qm_memory_uninit(struct hisi_qm *qm) +{ + uint32_t i = 0; + + for (i = 0; i < qm->qp_num; i++) + qp_memory_uninit(qm, i); + + free(qm->qp_array); + free(qm->sqc); + free(qm->cqc); +} + +static enum hisi_drv_status qm_memory_init(struct hisi_qm *qm) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + size_t sqc_size = 0; + size_t cqc_size = 0; + size_t qp_size = 0; + uint32_t i = 0; + int32_t j = 0; + + sqc_size = sizeof(struct qm_sqc) * qm->qp_num; + cqc_size = sizeof(struct qm_cqc) * qm->qp_num; + qp_size = sizeof(struct hisi_qp) * qm->qp_num; + + qm->sqc = memalign(HISI_QM_ALIGN32, sqc_size); + if (!qm->sqc) { + EMSG("Fail to malloc sqc"); + return HISI_QM_DRVCRYPT_ENOMEM; + } + qm->sqc_dma = virt_to_phys(qm->sqc); + + qm->cqc = memalign(HISI_QM_ALIGN32, cqc_size); + if (!qm->cqc) { + EMSG("Fail to malloc cqc"); + ret = HISI_QM_DRVCRYPT_ENOMEM; + goto free_sqc; + } + qm->cqc_dma = virt_to_phys(qm->cqc); + + qm->qp_array = (struct hisi_qp *)malloc(qp_size); + if (!qm->qp_array) { + EMSG("Fail to malloc qp_array"); + ret = HISI_QM_DRVCRYPT_ENOMEM; + goto free_cqc; + } + + for (i = 0; i < qm->qp_num; i++) { + ret = qp_memory_init(qm, i); + if (ret) { + ret = HISI_QM_DRVCRYPT_ENOMEM; + goto free_qp_mem; + } + } + + return HISI_QM_DRVCRYPT_NO_ERR; + +free_qp_mem: + for (j = (int32_t)i - 1; j >= 0; j--) + qp_memory_uninit(qm, j); + free(qm->qp_array); +free_cqc: + free(qm->cqc); +free_sqc: + free(qm->sqc); + return ret; +} + +enum hisi_drv_status hisi_qm_init(struct hisi_qm *qm) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + + if (qm->fun_type == HISI_QM_HW_VF) { + ret = qm_get_vft(qm, &qm->qp_base, &qm->qp_num); + if (ret) { + EMSG("Fail to get function vft config"); + return ret; + } + } + + if (qm->qp_num == 0 || qm->sqe_size == 0) { + EMSG("Invalid qm parameters"); + return HISI_QM_DRVCRYPT_EINVAL; + } + + ret = qm_memory_init(qm); + if (ret) + return ret; + + qm->qp_in_used = 0; + qm->qp_idx = 0; + mutex_init(&qm->qp_lock); + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static void qm_cache_writeback(struct hisi_qm *qm) +{ + uint32_t val = 0; + + io_write32(qm->io_base + QM_CACHE_WB_START, QM_FVT_CFG_RDY_BIT); + + if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE, val, + val & QM_FVT_CFG_RDY_BIT, POLL_PERIOD, + POLL_TIMEOUT)) + panic("QM writeback sqc cache fail"); +} + +void hisi_qm_uninit(struct hisi_qm *qm) +{ + qm_cache_writeback(qm); + qm_memory_uninit(qm); + mutex_destroy(&qm->qp_lock); +} + +static enum hisi_drv_status qm_hw_mem_reset(struct hisi_qm *qm) +{ + uint32_t val = 0; + + io_write32(qm->io_base + QM_MEM_START_INIT, QM_FVT_CFG_RDY_BIT); + + return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val, + val & QM_FVT_CFG_RDY_BIT, POLL_PERIOD, + POLL_TIMEOUT); +} + +static enum hisi_drv_status qm_func_vft_cfg(struct hisi_qm *qm) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + uint32_t q_base = qm->qp_num; + uint32_t act_q_num = 0; + uint32_t i = 0; + uint32_t j = 0; + + if (qm->vfs_num == 0) + return HISI_QM_DRVCRYPT_NO_ERR; + + if (qm->vfs_num > HISI_QM_MAX_VFS_NUM) { + EMSG("Invalid QM vfs_num"); + return HISI_QM_DRVCRYPT_EINVAL; + } + + for (i = 1; i <= qm->vfs_num; i++) { + act_q_num = HISI_QM_VF_Q_NUM; + ret = qm_set_xqc_vft(qm, i, q_base, act_q_num); + if (ret) { + for (j = 1; j < i; j++) + (void)qm_set_xqc_vft(qm, j, 0, 0); + return ret; + } + q_base += act_q_num; + } + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +enum hisi_drv_status hisi_qm_start(struct hisi_qm *qm) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + + if (qm->fun_type == HISI_QM_HW_PF) { + ret = qm_hw_mem_reset(qm); + if (ret) { + EMSG("Fail to reset qm hardware mem"); + return ret; + } + + ret = qm_set_xqc_vft(qm, 0, qm->qp_base, qm->qp_num); + if (ret) { + EMSG("Fail to set PF xqc_vft"); + return ret; + } + + ret = qm_func_vft_cfg(qm); + if (ret) { + EMSG("Fail to set VF xqc_vft"); + return ret; + } + } + + ret = qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, QM_MB_OP_WR); + if (ret) { + EMSG("Fail to set sqc_bt"); + return ret; + } + + ret = qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, QM_MB_OP_WR); + if (ret) { + EMSG("Fail to set cqc_bt"); + return ret; + } + + /* security mode does not support msi */ + io_write32(qm->io_base + QM_VF_AEQ_INT_MASK, QM_VF_AEQ_INT_MASK_EN); + io_write32(qm->io_base + QM_VF_EQ_INT_MASK, QM_VF_EQ_INT_MASK_EN); + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +void hisi_qm_dev_init(struct hisi_qm *qm) +{ + if (qm->fun_type == HISI_QM_HW_VF) + return; + + /* qm user domain */ + io_write32(qm->io_base + QM_ARUSER_M_CFG_1, QM_AXUSER_CFG); + io_write32(qm->io_base + QM_ARUSER_M_CFG_ENABLE, AXUSER_M_CFG_ENABLE); + io_write32(qm->io_base + QM_AWUSER_M_CFG_1, QM_AXUSER_CFG); + io_write32(qm->io_base + QM_AWUSER_M_CFG_ENABLE, AXUSER_M_CFG_ENABLE); + /* qm cache */ + io_write32(qm->io_base + QM_AXI_M_CFG, AXI_M_CFG); + + if (qm->version == HISI_QM_HW_V2) { + /* disable FLR triggered by BME(bus master enable) */ + io_write32(qm->io_base + QM_PEH_AXUSER_CFG, PEH_AXUSER_CFG); + /* set sec sqc and cqc cache wb threshold 4 */ + io_write32(qm->io_base + QM_CACHE_CTL, QM_CACHE_CFG); + } + /* disable qm ras */ + io_write32(qm->io_base + HISI_QM_ABNML_INT_MASK, + HISI_QM_ABNML_INT_MASK_CFG); +} + +static enum hisi_drv_status qm_sqc_cfg(struct hisi_qp *qp) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + struct hisi_qm *qm = qp->qm; + struct qm_sqc *sqc = NULL; + paddr_t sqc_dma = 0; + + sqc = memalign(HISI_QM_ALIGN32, sizeof(struct qm_sqc)); + if (!sqc) + return HISI_QM_DRVCRYPT_ENOMEM; + + sqc_dma = virt_to_phys(sqc); + + memzero_explicit(sqc, sizeof(struct qm_sqc)); + reg_pair_from_64(qp->sqe_dma, &sqc->base_h, &sqc->base_l); + sqc->dw3 = (HISI_QM_Q_DEPTH - 1) | + SHIFT_U32(qm->sqe_log2_size, QM_SQ_SQE_SIZE_SHIFT); + sqc->rand_data = QM_DB_RAND_DATA; + sqc->cq_num = qp->qp_id; + sqc->w13 = BIT32(QM_SQ_ORDER_SHIFT) | + SHIFT_U32(qp->sq_type, QM_SQ_TYPE_SHIFT); + + ret = qm_mb(qm, QM_MB_CMD_SQC, sqc_dma, qp->qp_id, QM_MB_OP_WR); + free(sqc); + + return ret; +} + +static enum hisi_drv_status qm_cqc_cfg(struct hisi_qp *qp) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + struct hisi_qm *qm = qp->qm; + struct qm_cqc *cqc = NULL; + paddr_t cqc_dma = 0; + + cqc = memalign(HISI_QM_ALIGN32, sizeof(struct qm_cqc)); + if (!cqc) + return HISI_QM_DRVCRYPT_ENOMEM; + + cqc_dma = virt_to_phys(cqc); + + memzero_explicit(cqc, sizeof(struct qm_cqc)); + reg_pair_from_64(qp->cqe_dma, &cqc->base_h, &cqc->base_l); + cqc->dw3 = (HISI_QM_Q_DEPTH - 1) | + SHIFT_U32(QM_CQE_SIZE, QM_CQ_CQE_SIZE_SHIFT); + cqc->rand_data = QM_DB_RAND_DATA; + cqc->dw6 = PHASE_DEFAULT_VAL; + + ret = qm_mb(qm, QM_MB_CMD_CQC, cqc_dma, qp->qp_id, QM_MB_OP_WR); + free(cqc); + + return ret; +} + +struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, uint8_t sq_type) +{ + struct hisi_qp *qp = NULL; + + mutex_lock(&qm->qp_lock); + if (qm->qp_in_used == qm->qp_num) { + EMSG("All %"PRIu32" queues of QM are busy", qm->qp_num); + goto err_proc; + } + + if (qm->qp_idx == qm->qp_num - 1) + qm->qp_idx = 0; + else + qm->qp_idx++; + + qp = &qm->qp_array[qm->qp_idx]; + memzero_explicit(qp->cqe, sizeof(struct qm_cqe) * HISI_QM_Q_DEPTH); + qp->sq_type = sq_type; + qp->sq_tail = 0; + qp->cq_head = 0; + qp->cqc_phase = true; + + if (qm_sqc_cfg(qp)) { + EMSG("Fail to set qp[%"PRIu32"] sqc", qp->qp_id); + goto err_proc; + } + + if (qm_cqc_cfg(qp)) { + EMSG("Fail to set qp[%"PRIu32"] cqc", qp->qp_id); + goto err_proc; + } + + qm->qp_in_used++; + mutex_unlock(&qm->qp_lock); + return qp; + +err_proc: + mutex_unlock(&qm->qp_lock); + return NULL; +} + +void hisi_qm_release_qp(struct hisi_qp *qp) +{ + struct hisi_qm *qm = NULL; + + if (!qp) { + EMSG("qp is NULL"); + return; + } + + qm = qp->qm; + mutex_lock(&qm->qp_lock); + qm->qp_in_used--; + mutex_unlock(&qm->qp_lock); +} + +static void qm_sq_tail_update(struct hisi_qp *qp) +{ + if (qp->sq_tail == HISI_QM_Q_DEPTH - 1) + qp->sq_tail = 0; + else + qp->sq_tail++; +} + +/* + * One task thread will just bind to one hardware queue, and + * hardware does not support msi. So we have no lock here. + */ +enum hisi_drv_status hisi_qp_send(struct hisi_qp *qp, void *msg) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + struct hisi_qm *qm = NULL; + void *sqe = NULL; + + if (!qp) { + EMSG("qp is NULL"); + return HISI_QM_DRVCRYPT_EINVAL; + } + + qm = qp->qm; + ret = qm->dev_status_check(qm); + if (ret) + return ret; + + sqe = (void *)((vaddr_t)qp->sqe + qm->sqe_size * qp->sq_tail); + memzero_explicit(sqe, qm->sqe_size); + + ret = qp->fill_sqe(sqe, msg); + if (ret) { + EMSG("Fail to fill sqe"); + return ret; + } + + qm_sq_tail_update(qp); + + dsb(); + qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_SQ, qp->sq_tail, 0); + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static void qm_cq_head_update(struct hisi_qp *qp) +{ + if (qp->cq_head == HISI_QM_Q_DEPTH - 1) { + qp->cqc_phase = !qp->cqc_phase; + qp->cq_head = 0; + } else { + qp->cq_head++; + } +} + +#define HISI_QM_RECV_DONE 0xAF +static enum hisi_drv_status hisi_qp_recv(struct hisi_qp *qp, void *msg) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + struct hisi_qm *qm = qp->qm; + struct qm_cqe *cqe = NULL; + void *sqe = NULL; + + ret = qm->dev_status_check(qm); + if (ret) + return ret; + + cqe = qp->cqe + qp->cq_head; + if (QM_CQE_PHASE(cqe) == qp->cqc_phase) { + dsb_osh(); + sqe = (void *)((vaddr_t)qp->sqe + qm->sqe_size * cqe->sq_head); + ret = qp->parse_sqe(sqe, msg); + qm_cq_head_update(qp); + qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, qp->cq_head, 0); + if (ret) { + EMSG("Fail to parse sqe"); + return ret; + } + } else { + return HISI_QM_DRVCRYPT_NO_ERR; + } + + return HISI_QM_RECV_DONE; +} + +static void qm_dfx_dump(struct hisi_qm *qm) +{ + const struct qm_dfx_registers *regs = qm_dfx_regs; + __maybe_unused uint32_t val = 0; + + if (qm->fun_type == HISI_QM_HW_VF) + return; + + while (regs->reg_name) { + val = io_read32(qm->io_base + regs->reg_offset); + EMSG("%s= 0x%" PRIx32, regs->reg_name, val); + regs++; + } +} + +enum hisi_drv_status hisi_qp_recv_sync(struct hisi_qp *qp, void *msg) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + uint32_t cnt = 0; + + if (!qp) { + EMSG("qp is NULL"); + return HISI_QM_DRVCRYPT_EINVAL; + } + + while (true) { + ret = hisi_qp_recv(qp, msg); + if (ret == HISI_QM_DRVCRYPT_NO_ERR) { + if (++cnt > HISI_QM_RECV_SYNC_TIMEOUT) { + EMSG("qm recv task timeout"); + qm_dfx_dump(qp->qm); + return HISI_QM_DRVCRYPT_ETMOUT; + } + } else { + if (ret == HISI_QM_RECV_DONE) + return HISI_QM_DRVCRYPT_NO_ERR; + + EMSG("qm recv task error"); + qm_dfx_dump(qp->qm); + break; + } + } + + return ret; +} diff --git a/core/drivers/crypto/hisilicon/include/hisi_qm.h b/core/drivers/crypto/hisilicon/include/hisi_qm.h new file mode 100644 index 00000000000..344903dfcfa --- /dev/null +++ b/core/drivers/crypto/hisilicon/include/hisi_qm.h @@ -0,0 +1,276 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, Huawei Technologies Co., Ltd + */ +#ifndef __HISI_QM_H__ +#define __HISI_QM_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HISI_QM_HW_V2 0x21 +#define HISI_QM_HW_V3 0x30 +#define HISI_QM_MAX_VFS_NUM 63 +#define HISI_QM_PF_Q_BASE 0 +#define HISI_QM_PF_Q_NUM 64 +#define HISI_QM_VF_Q_NUM 15 +#define HISI_QM_Q_DEPTH 8 +#define PHASE_DEFAULT_VAL 0x1 + +#define HISI_QM_ABNML_INT_MASK 0x100004 +#define HISI_QM_ABNML_INT_MASK_CFG 0x7fff +#define HISI_QM_ABNML_INT_SRC 0x100000 +#define HISI_QM_HPRE_NFE_INT_MASK 0x6fb7 +#define HISI_QM_SEC_NFE_INT_MASK 0x6ff7 +#define HISI_QM_INVALID_DB BIT(12) +#define HISI_QM_REVISON_ID_BASE 0x1000dc +#define HISI_QM_REVISON_ID_MASK GENMASK_32(7, 0) +#define POLL_PERIOD 10 +#define POLL_TIMEOUT 1000 +#define HISI_QM_RECV_SYNC_TIMEOUT 0xfffffff +#define HISI_QM_ALIGN128 128 +#define HISI_QM_ALIGN32 32 + +enum qm_fun_type { + HISI_QM_HW_PF, + HISI_QM_HW_VF, +}; + +enum qm_sq_type { + HISI_QM_CHANNEL_TYPE0 = 0, + HISI_QM_CHANNEL_TYPE1, + HISI_QM_CHANNEL_TYPE2, +}; + +struct qm_sqc { + uint16_t head; + uint16_t tail; + uint32_t base_l; + uint32_t base_h; + /* + * qes : 12 + * sqe : 4 + * rsv(stash_nid/stash_en) : 16 + */ + uint32_t dw3; + uint16_t rand_data; + uint16_t rsv0; + uint16_t pasid; + /* + * rsv : 5 + * head_sig : 1 + * tail_sig : 1 + * pasid_en : 1 + * rsv : 8 + */ + uint16_t w11; + uint16_t cq_num; + /* + * priority(Credit): 4 + * order(order/fc/close/rsv) : 4 + * type : 4 + * rsv : 4 + */ + uint16_t w13; + uint32_t rsv1; +}; + +struct qm_cqc { + uint16_t head; + uint16_t tail; + uint32_t base_l; + uint32_t base_h; + /* + * qes : 12 + * cqe_size : 4 + * rsv(stash_nid/stash_en) : 16 + */ + uint32_t dw3; + uint16_t rand_data; + uint16_t rsv0; + uint16_t pasid; + /* + * pasid_en : 1 + * rsv : 4 + * head_sig : 1 + * tail_sig : 1 + * rsv : 9 + */ + uint16_t w11; + /* + * phase : 1 + * c_flag : 1 + * stash_vld : 1 + */ + uint32_t dw6; + uint32_t rsv1; +}; + +struct qm_cqe { + uint32_t rsv0; + uint16_t cmd_id; + uint16_t rsv1; + uint16_t sq_head; + uint16_t sq_id; + uint16_t rsv2; + /* + * p : 1 + * status : 15 + */ + uint16_t w7; +}; + +struct hisi_qp { + struct hisi_qm *qm; + uint32_t qp_id; + uint8_t sq_type; + uint16_t sq_tail; + uint16_t cq_head; + bool cqc_phase; + + void *sqe; + struct qm_cqe *cqe; + paddr_t sqe_dma; + paddr_t cqe_dma; + + enum hisi_drv_status (*fill_sqe)(void *sqe, void *msg); + enum hisi_drv_status (*parse_sqe)(void *sqe, void *msg); +}; + +struct hisi_qm { + enum qm_fun_type fun_type; + vaddr_t io_base; + uint32_t io_size; + uint32_t vfs_num; + uint32_t version; + + struct qm_sqc *sqc; + struct qm_cqc *cqc; + paddr_t sqc_dma; + paddr_t cqc_dma; + uint32_t sqe_size; + uint32_t sqe_log2_size; + uint32_t qp_base; + uint32_t qp_num; + uint32_t qp_in_used; + uint32_t qp_idx; + struct hisi_qp *qp_array; + struct mutex qp_lock; /* protect the qp instance */ + + enum hisi_drv_status (*dev_status_check)(struct hisi_qm *qm); +}; + +enum hisi_drv_status { + HISI_QM_DRVCRYPT_NO_ERR = 0, + HISI_QM_DRVCRYPT_FAIL = 1, + HISI_QM_DRVCRYPT_EIO = 5, + HISI_QM_DRVCRYPT_EAGAIN = 11, + HISI_QM_DRVCRYPT_ENOMEM = 12, + HISI_QM_DRVCRYPT_EFAULT = 14, + HISI_QM_DRVCRYPT_EBUSY = 16, + HISI_QM_DRVCRYPT_ENODEV = 19, + HISI_QM_DRVCRYPT_EINVAL = 22, + HISI_QM_DRVCRYPT_ETMOUT = 110, + HISI_QM_DRVCRYPT_ENOPROC, + HISI_QM_DRVCRYPT_IN_EPARA, + HISI_QM_DRVCRYPT_VERIFY_ERR, + HISI_QM_DRVCRYPT_HW_EACCESS, +}; + +#define readl_relaxed_poll_timeout(addr, val, cond, delay_us, timeout_us) \ +({ \ + uint64_t timeout = 0; \ + uint64_t _delay_us = delay_us; \ + bool flag = false; \ + while (timeout < (timeout_us)) { \ + (val) = io_read32(addr); \ + if (cond) { \ + flag = true; \ + break; \ + } \ + timeout += (_delay_us); \ + udelay(_delay_us); \ + } \ + (flag) ? HISI_QM_DRVCRYPT_NO_ERR : HISI_QM_DRVCRYPT_ETMOUT; \ +}) + +struct acc_device { + struct hisi_qm qm; + vaddr_t io_base; + uint32_t io_size; + uint32_t vfs_num; + uint32_t endian; + enum qm_fun_type fun_type; + SLIST_ENTRY(acc_device) link; +}; + +/** + *@Description: Get the version information of QM hardware + *@param qm: Handle of Queue Management module + */ +void hisi_qm_get_version(struct hisi_qm *qm); + +/** + *@Description: Init QM for Kunpeng drv + *@param qm: Handle of Queue Management module + *@return success: 0，fail: HISI_QM_DRVCRYPT_EBUSY/HISI_QM_DRVCRYPT_EINVAL + */ +enum hisi_drv_status hisi_qm_init(struct hisi_qm *qm); + +/** + *@Description:deinit QM for Kunpeng drv + *@param qm: Handle of Queue Management module + */ +void hisi_qm_uninit(struct hisi_qm *qm); + +/** + *@Description: Start QM for Kunpeng drv + *@param qm: Handle of Queue Management module + */ +enum hisi_drv_status hisi_qm_start(struct hisi_qm *qm); + +/** + *@Description: Config QM for Kunpeng drv + *@param qm: Handle of Queue Management module + */ +void hisi_qm_dev_init(struct hisi_qm *qm); + +/** + *@Description: Create Queue Pair, allocated to PF/VF for configure + * and service use. Each QP includes one SQ and one CQ + *@param qm: Handle of Queue Management module + *@param sq_type: Accelerator specific algorithm type in sqc + *@return success: Handle of QP，fail: NULL + */ +struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, uint8_t sq_type); + +/** + *@Description:Release Queue Pair + *@param qp: Handle of Queue Pair + */ +void hisi_qm_release_qp(struct hisi_qp *qp); + +/** + *@Description: Send SQE(Submmision Queue Element) to Kunpeng dev + *@param qp: Handle of Queue Pair + *@param msg: The message + *@return success: 0，fail: HISI_QM_DRVCRYPT_EINVAL + */ +enum hisi_drv_status hisi_qp_send(struct hisi_qp *qp, void *msg); + +/** + *@Description: Recevice result from Kunpeng dev + *@param qp: Handle of Queue Pair + *@param msg: The message + *@return success: 0，fail: HISI_QM_DRVCRYPT_EINVAL/ETMOUT + */ +enum hisi_drv_status hisi_qp_recv_sync(struct hisi_qp *qp, void *msg); + +#endif diff --git a/core/drivers/crypto/hisilicon/sub.mk b/core/drivers/crypto/hisilicon/sub.mk new file mode 100644 index 00000000000..9a1747084ff --- /dev/null +++ b/core/drivers/crypto/hisilicon/sub.mk @@ -0,0 +1,3 @@ +incdirs-y += include + +srcs-y += hisi_qm.c diff --git a/core/drivers/crypto/sub.mk b/core/drivers/crypto/sub.mk index 3c26eda79a2..71cb6bd6905 100644 --- a/core/drivers/crypto/sub.mk +++ b/core/drivers/crypto/sub.mk @@ -11,3 +11,5 @@ subdirs-$(CFG_STM32_CRYPTO_DRIVER) += stm32 subdirs-$(CFG_ASPEED_CRYPTO_DRIVER) += aspeed subdirs-$(CFG_VERSAL_CRYPTO_DRIVER) += versal + +subdirs-$(CFG_HISILICON_CRYPTO_DRIVER) += hisilicon From f8a7eef8df48fbf5740275d9c8fa1a96e6f6a68e Mon Sep 17 00:00:00 2001 From: Xiaoxu Zeng Date: Thu, 6 Jul 2023 19:49:29 +0800 Subject: [PATCH 2/3] drivers: crypto: hisilicon: implement SEC driver The Hisilicon SEC (Security Engine) is a hardware security acceleration module, which is used to enhance the security-related functions of the processor. Signed-off-by: Xiaoxu Zeng --- core/arch/arm/plat-d06/main.c | 1 + core/arch/arm/plat-d06/platform_config.h | 4 + .../crypto/hisilicon/include/hisi_sec.h | 347 ++++++++++++++++++ core/drivers/crypto/hisilicon/sec/hisi_sec.c | 270 ++++++++++++++ core/drivers/crypto/hisilicon/sec/sub.mk | 3 + core/drivers/crypto/hisilicon/sub.mk | 1 + 6 files changed, 626 insertions(+) create mode 100644 core/drivers/crypto/hisilicon/include/hisi_sec.h create mode 100644 core/drivers/crypto/hisilicon/sec/hisi_sec.c create mode 100644 core/drivers/crypto/hisilicon/sec/sub.mk diff --git a/core/arch/arm/plat-d06/main.c b/core/arch/arm/plat-d06/main.c index 77369440117..8c36bdd0214 100644 --- a/core/arch/arm/plat-d06/main.c +++ b/core/arch/arm/plat-d06/main.c @@ -10,6 +10,7 @@ static struct lpc_uart_data console_data __nex_bss; register_phys_mem_pgdir(MEM_AREA_IO_NSEC, LPC_BASE, LPC_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, SEC_BASE, SEC_SIZE); void console_init(void) { diff --git a/core/arch/arm/plat-d06/platform_config.h b/core/arch/arm/plat-d06/platform_config.h index 81979bdc478..de82a802f98 100644 --- a/core/arch/arm/plat-d06/platform_config.h +++ b/core/arch/arm/plat-d06/platform_config.h @@ -17,4 +17,8 @@ #define CONSOLE_BAUDRATE 115200 #define CONSOLE_UART_CLK_IN_HZ 200 +/* SEC */ +#define SEC_BASE 0x141800000 +#define SEC_SIZE 0x400000 + #endif /* PLATFORM_CONFIG_H */ diff --git a/core/drivers/crypto/hisilicon/include/hisi_sec.h b/core/drivers/crypto/hisilicon/include/hisi_sec.h new file mode 100644 index 00000000000..0e57540d533 --- /dev/null +++ b/core/drivers/crypto/hisilicon/include/hisi_sec.h @@ -0,0 +1,347 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, HiSilicon Limited + */ +#ifndef __HISI_SEC_H__ +#define __HISI_SEC_H__ + +#include "hisi_qm.h" + +/* + * Version number maintenance rule: + * first.second.third: year.month.change num + */ +#define SEC_MODULE_VERSION "22.11.2" + +#ifdef CFG_HISILICON_ACC_V3 +#define SEC_BAR 0x160000000 +#else +#define SEC_BAR 0x141800000 +#endif +#define SEC_SIZE 0x400000 + +#define SEC_SQE_SIZE 128 +#define SEC_SQE_LOG2_SIZE 7 + +#define SEC_PF_ABNORMAL_INT_SOURCE_REG 0x0010 +#define SEC_NFE_ERROR_MASK 0x24 +#define SEC_SQE_ICV_SHIFT 1 +#define SEC_SQE_FLAG_SHIFT 7 +#define SEC_MAC_TO_DDR 0x1 +#define SEC_PBKDF2 0x8 +#define MAC_LEN 0x8 +#define MAC_LEN_HMAC_SHA224 0x7 +#define AUTH_KEY_LEN 0x8 +#define C_NUM 10000 +#define PASS_WORD_LEN 0x20 +#define GET_IMG_ROTKEY_AP 0x6 +#define SEC_PBKDF2_SUCC 0x81 +#define SEC_HW_TASK_DONE 0x1 +#define SQE_BYTES_NUMS 128 +#define SEC_USE_HUK 0x1 +#define AES_KEYSIZE_128 16 +#define AES_KEYSIZE_192 24 +#define AES_KEYSIZE_256 32 +#define XTS_KEYSIZE_256 64 +#define XTS_KEYSIZE_128 32 +#define XTS_CKEY_LEN_128_BIT 0 +#define XTS_CKEY_LEN_256_BIT 2 + +#define TYPE_ENCRYPTIN 0 +#define SEC_CIPHER_THEN_DIGEST 0 +#define SEC_DIGEST_THEN_CIPHER 1 + +#define ECB_CBC_SRC_ALIGN_MASK 0xf +#define CTR_SRC_ALIGN_MASK 0xf +#define CTR_SRC_BLOCK_SIZE 0x10 +#define BYTE_BITS 0x8 + +struct hisi_sec_sqe_type2 { + uint32_t nonce_len : 4; + uint32_t huk : 1; + uint32_t key_s : 1; + uint32_t ci_gen : 2; + uint32_t ai_gen : 2; + uint32_t a_pad : 2; + uint32_t c_s : 2; + uint32_t rsvd1 : 2; + uint32_t rhf : 1; + uint32_t c_key_type : 2; + uint32_t a_key_type : 2; + uint32_t write_frame_len : 3; + uint32_t cal_iv_addr_en : 1; + uint32_t tls_up : 1; + uint32_t rsvd0 : 5; + uint32_t inveld : 1; + uint32_t mac_len : 5; + uint32_t a_key_len : 6; + uint32_t a_alg : 6; + uint32_t rsvd3 : 15; + uint32_t c_icv_len : 6; + uint32_t c_width : 3; + uint32_t c_key_len : 3; + uint32_t c_mode : 4; + uint32_t c_alg : 4; + uint32_t rsvd4 : 12; + uint32_t a_len : 24; + uint32_t iv_offset_l : 8; + uint32_t c_len : 24; + uint32_t iv_offset_h : 8; + uint32_t auth_src_offset : 16; + uint32_t cipher_src_offset : 16; + uint32_t cs_ip_header_offset : 16; + uint32_t cs_udp_header_offset : 16; + uint32_t pass_word_len : 16; + uint32_t dk_len : 16; + uint32_t salt3 : 8; + uint32_t salt2 : 8; + uint32_t salt1 : 8; + uint32_t salt0 : 8; + uint32_t tag : 16; + uint32_t rsvd5 : 16; + uint32_t c_pad_type : 4; + uint32_t c_pad_len : 8; + uint32_t c_pad_data_type : 4; + uint32_t c_pad_len_field : 2; + uint32_t rsvd6 : 14; + uint32_t long_a_data_len_l; + uint32_t long_a_data_len_h; + uint32_t a_ivin_addr_l; + uint32_t a_ivin_addr_h; + uint32_t a_key_addr_l; + uint32_t a_key_addr_h; + uint32_t mac_addr_l; + uint32_t mac_addr_h; + uint32_t c_ivin_addr_l; + uint32_t c_ivin_addr_h; + uint32_t c_key_addr_l; + uint32_t c_key_addr_h; + uint32_t data_src_addr_l; + uint32_t data_src_addr_h; + uint32_t data_dst_addr_l; + uint32_t data_dst_addr_h; + uint32_t done : 1; + uint32_t icv : 3; + uint32_t rsvd11 : 3; + uint32_t flag : 4; + uint32_t rsvd10 : 5; + uint32_t error_type : 8; + uint32_t warning_type : 8; + uint32_t mac_i3 : 8; + uint32_t mac_i2 : 8; + uint32_t mac_i1 : 8; + uint32_t mac_i0 : 8; + uint32_t check_sum_i : 16; + uint32_t tls_pad_len_i : 8; + uint32_t rsvd12 : 8; + uint32_t counter; +}; + +struct hisi_sec_sqe { + uint32_t type : 4; + uint32_t cipher : 2; + uint32_t auth : 2; + uint32_t seq : 1; + uint32_t de : 2; + uint32_t scene : 4; + uint32_t src_addr_type : 3; + uint32_t dst_addr_type : 3; + uint32_t mac_addr_type : 3; + uint32_t rsvd0 : 8; + struct hisi_sec_sqe_type2 type2; /* the other sense */ +}; + +struct bd3_auth_key_iv { + uint32_t a_key_addr_l; + uint32_t a_key_addr_h; + uint32_t a_ivin_addr_l; + uint32_t a_ivin_addr_h; + uint32_t rsvd0; + uint32_t rsvd1; +}; + +struct bd3_ipsec_scene { + uint32_t c_ivin_addr_l; + uint32_t c_ivin_addr_h; + uint32_t c_s : 2; + uint32_t deal_esp_ah : 4; + uint32_t protocol_type : 4; + uint32_t mode : 2; + uint32_t ip_type : 2; + uint32_t mac_sel : 1; + uint32_t rsvd0 : 1; + uint32_t next_header : 8; + uint32_t pad_len : 8; + uint32_t iv_offset : 16; + uint32_t rsvd1 : 16; + uint32_t cs_ip_header_offset : 16; + uint32_t cs_udp_header_offset : 16; +}; + +struct bd3_pbkdf2_scene { + uint32_t c_ivin_addr_l; + uint32_t c_ivin_addr_h; + uint32_t pbkdf2_salt_len : 24; + uint32_t rsvd0 : 8; + uint32_t c_num : 24; + uint32_t rsvd1 : 8; + uint32_t pass_word_len : 16; + uint32_t dk_len : 16; +}; + +struct bd3_stream_scene { + uint32_t c_ivin_addr_l; + uint32_t c_ivin_addr_h; + uint32_t long_a_data_len_l; + uint32_t long_a_data_len_h; + uint32_t auth_pad : 2; + uint32_t stream_protocol : 3; + uint32_t mac_sel : 1; + uint32_t rsvd0 : 2; + uint32_t plaintext_type : 8; + uint32_t pad_len_1p3 : 16; +}; + +struct bd3_check_sum { + uint32_t check_sum_i : 16; + uint32_t tls_pad_len_i : 8; + uint32_t rsvd0 : 8; +}; + +struct hisi_sec_bd3_sqe { + uint32_t type : 4; + uint32_t inveld : 1; + uint32_t scene : 4; + uint32_t de : 2; + uint32_t src_addr_type : 3; + uint32_t dst_addr_type : 3; + uint32_t mac_addr_type : 3; + uint32_t rsvd : 12; + + uint32_t cipher : 2; + uint32_t ci_gen : 2; + uint32_t c_icv_len : 6; + uint32_t c_width : 3; + uint32_t c_key_len : 3; + uint32_t c_mode : 4; + uint32_t c_alg : 4; + uint32_t nonce_len : 4; + uint32_t rsv : 1; + uint32_t cal_iv_addr_en : 1; + uint32_t seq : 1; + uint32_t rsvd0 : 1; + + uint32_t tag_l; + uint32_t tag_h; + uint32_t data_src_addr_l; + uint32_t data_src_addr_h; + + struct bd3_auth_key_iv auth_key_iv; + + uint32_t c_key_addr_l; + uint32_t c_key_addr_h; + uint32_t auth : 2; + uint32_t ai_gen : 2; + uint32_t mac_len : 5; + uint32_t a_key_len : 6; + uint32_t a_alg : 6; + uint32_t key_sel : 4; + uint32_t ctr_counter_mode : 2; + uint32_t sva_prefetch : 1; + uint32_t key_wrap_num : 3; + uint32_t update_key : 1; + + uint32_t salt3 : 8; + uint32_t salt2 : 8; + uint32_t salt1 : 8; + uint32_t salt0 : 8; + uint32_t auth_src_offset : 16; + uint32_t cipher_src_offset : 16; + uint32_t a_len : 24; + uint32_t auth_key_offset : 8; + uint32_t c_len : 24; + uint32_t auth_ivin_offset : 8; + uint32_t data_dst_addr_l; + uint32_t data_dst_addr_h; + uint32_t mac_addr_l; + uint32_t mac_addr_h; + + union { + struct bd3_ipsec_scene ipsec_scene; + struct bd3_pbkdf2_scene pbkdf2_scene; + struct bd3_stream_scene stream_scene; + }; + + uint32_t done : 1; + uint32_t icv : 3; + uint32_t csc : 3; + uint32_t flag : 4; + uint32_t dc : 3; + uint32_t rsvd10 : 2; + uint32_t error_type : 8; + uint32_t warning_type : 8; + union { + uint32_t mac_i; + uint32_t kek_key_addr_l; + }; + union { + uint32_t kek_key_addr_h; + struct bd3_check_sum check_sum; + }; + uint32_t counter; +}; + +enum { + NO_CIPHER, + CIPHER_ENCRYPT, + CIPHER_DECRYPT, + REPORT_COPY, +}; + +enum sec_bd_type { + BD_TYPE1 = 0x1, + BD_TYPE2 = 0x2, + BD_TYPE3 = 0x3, +}; + +enum CKEY_LEN { + CKEY_LEN_128_BIT = 0x0, + CKEY_LEN_192_BIT = 0x1, + CKEY_LEN_256_BIT = 0x2, + CKEY_LEN_SM4 = 0x0, + CKEY_LEN_DES = 0x1, + CKEY_LEN_3DES_3KEY = 0x1, + CKEY_LEN_3DES_2KEY = 0x3, +}; + +enum { + SCENE_NOTHING = 0x0, + SCENE_IPSEC = 0x1, + SCENE_SSL_TLS = 0x3, + SCENE_DTLS = 0x4, + SCENE_STORAGE = 0x5, + SCENE_NAS = 0x6, + SCENE_STREAM = 0x7, + SCENE_PBKDF2 = 0x8, + SCENE_SMB = 0x9, +}; + +enum { + DATA_DST_ADDR_DISABLE, + DATA_DST_ADDR_ENABLE, +}; + +enum hisi_buff_type { + HISI_FLAT_BUF, + HISI_SGL_BUF, +}; + +/** + *@Description: Create Queue Pair for SEC, allocated to PF/VF for configure + * and service use. Each QP includes one SQ and one CQ + *@param sq_type: Accelerator specific algorithm type in sqc + *@return success: Handle of QP，fail: NULL + */ +struct hisi_qp *hisi_sec_create_qp(uint8_t sq_type); + +#endif diff --git a/core/drivers/crypto/hisilicon/sec/hisi_sec.c b/core/drivers/crypto/hisilicon/sec/hisi_sec.c new file mode 100644 index 00000000000..4be69909855 --- /dev/null +++ b/core/drivers/crypto/hisilicon/sec/hisi_sec.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 HiSilicon Limited. + * Kunpeng hardware accelerator sec module init. + */ +#include +#include "hisi_sec.h" + +#define AM_CFG_SINGLE_PORT_MAX_TRANS 0x300014 +#define SEC_CORE_INT_MASK 0x301000 +#define SEC_CORE_INT_SOURCE 0x301010 +#define SEC_RAS_CE_ENABLE 0x301050 +#define SEC_RAS_FE_ENABLE 0x301054 +#define SEC_RAS_NFE_ENABLE 0x301058 +#define SEC_MEM_START_INIT 0x301100 +#define SEC_MEM_INIT_DONE 0x301104 +#define SEC_CONTROL_REG 0x301200 +#define SEC_INTERFACE_USER_CTRL0 0x301220 +#define SEC_INTERFACE_USER_CTRL1 0x301224 +#define SEC_SAA_EN 0x301270 +#define SEC_BD_ERR_CHK_EN0 0x301380 +#define SEC_BD_ERR_CHK_EN1 0x301384 +#define SEC_BD_ERR_CHK_EN3 0x30138c +#define SEC_DYNAMIC_GATE_V3 0x30121c +#define SEC_CORE_AUTO_GATE_V3 0x30212c +#define SEC_INTERFACE_USER_CTRL0_V3 0x302220 +#define SEC_INTERFACE_USER_CTRL1_V3 0x302224 +#define SEC_SINGLE_PORT_MAX_TRANS 0x2060 +#define SEC_ABNML_INT_DISABLE 0x0 +#define SEC_RAS_CE_ENB_MASK 0x88 +#define SEC_RAS_FE_ENB_MASK 0x0 +#define SEC_RAS_NFE_ENB_MASK 0x177 +#define SEC_CLK_GATE_ENABLE BIT(3) +#define SEC_DYNAMIC_GATE_EN 0x7bff +#define SEC_CORE_AUTO_GATE_EN GENMASK_32(3, 0) +#define SEC_TRNG_EN_MASK BIT(8) +#define SEC_SAA_ENABLE 0x17f +#define SEC_SAA_ENABLE_V3 0xf +#define SEC_BD_ERR_CHK0 0xefffffff +#define SEC_BD_ERR_CHK1 0x7ffff7fd +#define SEC_BD_ERR_CHK3 0xffffbfff +#define SEC_USER0_CFG 0x20200 +#define SEC_USER0_SMMU_NORMAL (BIT(23) | BIT(15)) +#define SEC_USER1_CFG 0x12141214 +#define SEC_USER1_SMMU_NORMAL (BIT(31) | BIT(23) | BIT(15) | BIT(7)) +#define SEC_USER0_CFG_V3 0x20200 +#define SEC_USER1_CFG_V3 0x8c494 +#define SEC_LITTLE_ENDIAN 0 + +static SLIST_HEAD(, acc_device) sec_list = SLIST_HEAD_INITIALIZER(sec_list); + +struct hisi_qp *hisi_sec_create_qp(uint8_t sq_type) +{ + struct acc_device *sec_dev = NULL; + struct acc_device *cur_dev = NULL; + struct hisi_qm *qm = NULL; + uint32_t max_qp_num = 0; + uint32_t free_qp_num = 0; + + SLIST_FOREACH(cur_dev, &sec_list, link) { + qm = &cur_dev->qm; + free_qp_num = (qm->fun_type == HISI_QM_HW_PF ? + HISI_QM_PF_Q_NUM : HISI_QM_VF_Q_NUM) - qm->qp_in_used; + if (free_qp_num > max_qp_num) { + max_qp_num = free_qp_num; + sec_dev = cur_dev; + } + } + + if (!sec_dev) { + EMSG("No available sec device"); + return NULL; + } + + return hisi_qm_create_qp(&sec_dev->qm, sq_type); +} + +static void sec_disable_clock_gate(struct hisi_qm *qm) +{ + /* HISI_QM_HW_V2 version need to close clock gating */ + io_clrbits32(qm->io_base + SEC_CONTROL_REG, SEC_CLK_GATE_ENABLE); +} + +static void sec_enable_clock_gate(struct hisi_qm *qm) +{ + if (qm->version == HISI_QM_HW_V2) + return; + + io_setbits32(qm->io_base + SEC_CONTROL_REG, SEC_CLK_GATE_ENABLE); + io_write32(qm->io_base + SEC_DYNAMIC_GATE_V3, SEC_DYNAMIC_GATE_EN); + io_write32(qm->io_base + SEC_CORE_AUTO_GATE_V3, SEC_CORE_AUTO_GATE_EN); +} + +static enum hisi_drv_status sec_engine_init(struct acc_device *sec_dev) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + struct hisi_qm *qm = &sec_dev->qm; + uint32_t val = 0; + + if (qm->fun_type == HISI_QM_HW_VF) + return HISI_QM_DRVCRYPT_NO_ERR; + + sec_disable_clock_gate(qm); + hisi_qm_dev_init(qm); + + io_write32(qm->io_base + SEC_MEM_START_INIT, 0x1); + ret = readl_relaxed_poll_timeout(qm->io_base + SEC_MEM_INIT_DONE, val, + val & 0x1, POLL_PERIOD, POLL_TIMEOUT); + if (ret) { + EMSG("Fail to init sec mem"); + return ret; + } + + io_setbits32(qm->io_base + SEC_CONTROL_REG, SEC_TRNG_EN_MASK); + + if (qm->version == HISI_QM_HW_V2) { + /* smmu bypass */ + io_write32(qm->io_base + SEC_INTERFACE_USER_CTRL0, + SEC_USER0_CFG); + io_write32(qm->io_base + SEC_INTERFACE_USER_CTRL1, + SEC_USER1_CFG); + io_write32(qm->io_base + AM_CFG_SINGLE_PORT_MAX_TRANS, + SEC_SINGLE_PORT_MAX_TRANS); + io_write32(qm->io_base + SEC_SAA_EN, SEC_SAA_ENABLE); + /* HW V2 enable sm4 extra mode, as ctr/ecb */ + io_write32(qm->io_base + SEC_BD_ERR_CHK_EN0, SEC_BD_ERR_CHK0); + /* Enable sm4 xts mode multiple iv */ + io_write32(qm->io_base + SEC_BD_ERR_CHK_EN1, SEC_BD_ERR_CHK1); + io_write32(qm->io_base + SEC_BD_ERR_CHK_EN3, SEC_BD_ERR_CHK3); + } else { + /* cmd_type is controlled by hac subctrl, default normal */ + io_write32(qm->io_base + SEC_INTERFACE_USER_CTRL0_V3, + SEC_USER0_CFG_V3); + io_write32(qm->io_base + SEC_INTERFACE_USER_CTRL1_V3, + SEC_USER1_CFG_V3); + io_write32(qm->io_base + SEC_SAA_EN, SEC_SAA_ENABLE_V3); + } + + io_write32(qm->io_base + SEC_RAS_CE_ENABLE, SEC_RAS_CE_ENB_MASK); + io_write32(qm->io_base + SEC_RAS_FE_ENABLE, SEC_RAS_FE_ENB_MASK); + io_write32(qm->io_base + SEC_RAS_NFE_ENABLE, SEC_RAS_NFE_ENB_MASK); + io_write32(qm->io_base + SEC_CORE_INT_MASK, SEC_ABNML_INT_DISABLE); + + io_setbits32(qm->io_base + SEC_CONTROL_REG, sec_dev->endian); + + sec_enable_clock_gate(qm); + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static enum hisi_drv_status sec_dev_status_check(struct hisi_qm *qm) +{ + uint32_t val = 0; + + val = io_read32(qm->io_base + SEC_CORE_INT_SOURCE); + if (val & SEC_RAS_NFE_ENB_MASK) { + EMSG("SEC NFE RAS happened, need to reset"); + return HISI_QM_DRVCRYPT_HW_EACCESS; + } + + val = io_read32(qm->io_base + HISI_QM_ABNML_INT_SRC); + if (val) { + if (val & HISI_QM_SEC_NFE_INT_MASK) + EMSG("QM NFE RAS happened, need to reset"); + + if (val & HISI_QM_INVALID_DB) { + EMSG("QM invalid db happened, please check"); + io_write32(qm->io_base + HISI_QM_ABNML_INT_SRC, + HISI_QM_INVALID_DB); + } + + return HISI_QM_DRVCRYPT_HW_EACCESS; + } + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static enum hisi_drv_status sec_qm_init(struct acc_device *sec_dev) +{ + struct hisi_qm *qm = &sec_dev->qm; + + if (cpu_mmu_enabled()) { + qm->io_base = (vaddr_t)phys_to_virt_io(sec_dev->io_base, + sec_dev->io_size); + if (!qm->io_base) { + EMSG("Fail to get qm io_base"); + return HISI_QM_DRVCRYPT_EFAULT; + } + } else { + qm->io_base = sec_dev->io_base; + } + + qm->fun_type = sec_dev->fun_type; + qm->vfs_num = sec_dev->vfs_num; + qm->sqe_size = SEC_SQE_SIZE; + qm->sqe_log2_size = SEC_SQE_LOG2_SIZE; + if (qm->fun_type == HISI_QM_HW_PF) { + hisi_qm_get_version(qm); + IMSG("SEC hardware version is 0x%" PRIx32, qm->version); + qm->qp_base = HISI_QM_PF_Q_BASE; + qm->qp_num = HISI_QM_PF_Q_NUM; + qm->dev_status_check = sec_dev_status_check; + } + + return hisi_qm_init(qm); +} + +static struct acc_device *sec_pre_init(void) +{ + struct acc_device *sec_dev = NULL; + + sec_dev = (struct acc_device *)calloc(1, sizeof(*sec_dev)); + if (!sec_dev) { + EMSG("Fail to alloc sec_dev"); + return NULL; + } + + sec_dev->io_base = SEC_BAR; + sec_dev->io_size = SEC_SIZE; + sec_dev->vfs_num = 0; + sec_dev->endian = SEC_LITTLE_ENDIAN; + sec_dev->fun_type = HISI_QM_HW_PF; + SLIST_INSERT_HEAD(&sec_list, sec_dev, link); + + return sec_dev; +} + +static TEE_Result sec_probe(void) +{ + struct acc_device *sec_dev = NULL; + struct hisi_qm *qm = NULL; + TEE_Result ret = TEE_SUCCESS; + + IMSG("SEC driver init start, version %s", SEC_MODULE_VERSION); + sec_dev = sec_pre_init(); + if (!sec_dev) + return TEE_ERROR_STORAGE_NO_SPACE; + + qm = &sec_dev->qm; + ret = sec_qm_init(sec_dev); + if (ret) { + EMSG("Fail to init sec qm, ret=%d\n", ret); + goto err_with_pre_init; + } + + ret = sec_engine_init(sec_dev); + if (ret) { + EMSG("fail to init engine, ret=%d\n", ret); + goto err_with_qm_init; + } + + ret = hisi_qm_start(qm); + if (ret) { + EMSG("Fail to start qm, ret=%d\n", ret); + goto err_with_qm_init; + } + + IMSG("SEC driver init done"); + return TEE_SUCCESS; + +err_with_qm_init: + hisi_qm_uninit(qm); +err_with_pre_init: + SLIST_REMOVE_HEAD(&sec_list, link); + free(sec_dev); + + return TEE_ERROR_GENERIC; +} + +driver_init(sec_probe); diff --git a/core/drivers/crypto/hisilicon/sec/sub.mk b/core/drivers/crypto/hisilicon/sec/sub.mk new file mode 100644 index 00000000000..2f3e044ad41 --- /dev/null +++ b/core/drivers/crypto/hisilicon/sec/sub.mk @@ -0,0 +1,3 @@ +incdirs-y += ../include + +srcs-y += hisi_sec.c diff --git a/core/drivers/crypto/hisilicon/sub.mk b/core/drivers/crypto/hisilicon/sub.mk index 9a1747084ff..7cb3f6d4122 100644 --- a/core/drivers/crypto/hisilicon/sub.mk +++ b/core/drivers/crypto/hisilicon/sub.mk @@ -1,3 +1,4 @@ incdirs-y += include +subdirs-y += sec srcs-y += hisi_qm.c From 999da9d04a838d86b45f3d0ea43aa520f4d731a5 Mon Sep 17 00:00:00 2001 From: Xiaoxu Zeng Date: Thu, 6 Jul 2023 19:54:58 +0800 Subject: [PATCH 3/3] drivers: crypto: hisilicon: implement SEC ciphers Add the Hisilicon SEC as a drvcrypt cipher provider. Supported for DES-ECB/CBC 3DES-ECB/CBC AES-ECB/CBC/CTR SM4-ECB/CBC/CTR. Signed-off-by: Xiaoxu Zeng --- core/drivers/crypto/hisilicon/crypto.mk | 6 + .../crypto/hisilicon/include/hisi_cipher.h | 77 ++ .../drivers/crypto/hisilicon/sec/sec_cipher.c | 774 ++++++++++++++++++ core/drivers/crypto/hisilicon/sec/sub.mk | 1 + 4 files changed, 858 insertions(+) create mode 100644 core/drivers/crypto/hisilicon/crypto.mk create mode 100644 core/drivers/crypto/hisilicon/include/hisi_cipher.h create mode 100644 core/drivers/crypto/hisilicon/sec/sec_cipher.c diff --git a/core/drivers/crypto/hisilicon/crypto.mk b/core/drivers/crypto/hisilicon/crypto.mk new file mode 100644 index 00000000000..746f74c6511 --- /dev/null +++ b/core/drivers/crypto/hisilicon/crypto.mk @@ -0,0 +1,6 @@ +ifeq ($(CFG_HISILICON_CRYPTO_DRIVER),y) +$(call force,CFG_CRYPTO_DRIVER,y) +CFG_CRYPTO_DRIVER_DEBUG ?= 0 +$(call force,CFG_CRYPTO_DRV_CIPHER,y,Mandated by CFG_HISILICON_CRYPTO_DRIVER) +$(call force,CFG_CRYPTO_DRV_ACIPHER,y,Mandated by CFG_HISILICON_CRYPTO_DRIVER) +endif diff --git a/core/drivers/crypto/hisilicon/include/hisi_cipher.h b/core/drivers/crypto/hisilicon/include/hisi_cipher.h new file mode 100644 index 00000000000..1f31ceb8012 --- /dev/null +++ b/core/drivers/crypto/hisilicon/include/hisi_cipher.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, HiSilicon Limited + */ +#ifndef __HISI_CIPHER_H__ +#define __HISI_CIPHER_H__ + +#include +#include + +enum C_ALG { + C_ALG_DES = 0x0, + C_ALG_3DES = 0x1, + C_ALG_AES = 0x2, + C_ALG_SM4 = 0x3, +}; + +enum C_MODE { + C_MODE_ECB = 0x0, + C_MODE_CBC, + C_MODE_CFB, + C_MODE_OFB, + C_MODE_CTR, + C_MODE_CCM, + C_MODE_GCM, + C_MODE_XTS, + C_MODE_CBC_CS = 0x9, +}; + +#define DES_KEY_SIZE 8 +#define SEC_3DES_2KEY_SIZE (2 * DES_KEY_SIZE) +#define SEC_3DES_3KEY_SIZE (3 * DES_KEY_SIZE) +#define SEC_SM4_XTS_KEY_SIZE 32 +#define SEC_SM4_ECB_KEY_SIZE 16 +#define SEC_MAX_CIPHER_KEY_SIZE 64 +#define MAX_CIPHER_LENGTH 16776704 +#define MIN_CIPHER_LENGTH 16 + +#define DES_CBC_IV_SIZE 8 +#define AES_SM4_IV_SIZE 16 +#define SEC_MAX_IV_SIZE 16 +#define CTR_MODE_LEN_SHIFT 4 +#define CTR_128BIT_COUNTER 16 +#define AES_SM4_BLOCK_SIZE 16 +#define LEFT_MOST_BIT 7 + +static inline uint32_t multiple_round(uint32_t x, uint32_t y) +{ + uint32_t res = 0; + + if (ADD_OVERFLOW(x, y - 1, &res)) + res = UINT32_MAX; + + return res / y; +} + +struct sec_cipher_ctx { + uint8_t key[SEC_MAX_CIPHER_KEY_SIZE]; + uint8_t iv[SEC_MAX_IV_SIZE]; + uint64_t key_dma; + uint64_t iv_dma; + uint8_t *in; + uint64_t in_dma; + uint8_t *out; + uint64_t out_dma; + struct hisi_qp *qp; + size_t offs; + uint32_t len; + uint8_t alg; + uint8_t mode; + uint8_t iv_len; + uint8_t key_len; /* cipher key len */ + uint8_t c_key_len; /* cipher key type */ + bool encrypt; +}; + +#endif diff --git a/core/drivers/crypto/hisilicon/sec/sec_cipher.c b/core/drivers/crypto/hisilicon/sec/sec_cipher.c new file mode 100644 index 00000000000..ee0b1a7585c --- /dev/null +++ b/core/drivers/crypto/hisilicon/sec/sec_cipher.c @@ -0,0 +1,774 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, HiSilicon Limited + */ +#include +#include "hisi_cipher.h" +#include "hisi_sec.h" + +static TEE_Result sec_do_cipher_task(struct hisi_qp *qp, void *msg) +{ + TEE_Result ret = TEE_SUCCESS; + + ret = hisi_qp_send(qp, msg); + if (ret) { + EMSG("Fail to send task, ret=%"PRIu32, ret); + return TEE_ERROR_BAD_STATE; + } + + ret = hisi_qp_recv_sync(qp, msg); + if (ret) { + EMSG("Recv task error, ret=%"PRIu32, ret); + return TEE_ERROR_BAD_STATE; + } + + return TEE_SUCCESS; +} + +/* increment counter (128-bit int) by c */ +static void ctr_iv_inc(uint8_t *counter, uint32_t shift_len) +{ + uint32_t n = CTR_128BIT_COUNTER; + uint8_t *counter1 = counter; + uint32_t c = shift_len; + + do { + --n; + c += counter1[n]; + counter1[n] = (uint8_t)c; + c >>= BYTE_BITS; + } while (n); +} + +static void xts_multi_galois(unsigned char *data) +{ + uint8_t tt = 0; + uint8_t t = 0; + int i = 0; + + for (i = 0; i < AES_SM4_IV_SIZE; i++) { + tt = data[i] >> LEFT_MOST_BIT; + data[i] = ((data[i] << 1) | t) & 0xFF; + t = tt; + } + if (tt) + data[0] ^= 0x87; +} + +static TEE_Result sec_cipher_set_key(struct sec_cipher_ctx *c_ctx, + const uint8_t *key1, const int key1_len, + const uint8_t *key2, const int key2_len); + +/* + * When the IV is delivered by segment, the AES/SM4-ECB is used + * to update the IV to be used next time. + */ +static TEE_Result xts_iv_update(struct sec_cipher_ctx *c_ctx) +{ + size_t xts_key_len = c_ctx->key_len / 2; + struct sec_cipher_ctx ecb_ctx = { }; + TEE_Result ret = TEE_SUCCESS; + size_t i = 0; + + ecb_ctx.alg = c_ctx->alg; + ecb_ctx.mode = C_MODE_ECB; + ret = sec_cipher_set_key(&ecb_ctx, c_ctx->key + xts_key_len, + xts_key_len, NULL, 0); + if (ret) + return ret; + + ecb_ctx.encrypt = true; + ecb_ctx.in = c_ctx->iv; + ecb_ctx.out = c_ctx->iv; + ecb_ctx.in_dma = c_ctx->iv_dma; + ecb_ctx.out_dma = c_ctx->iv_dma; + ecb_ctx.len = c_ctx->iv_len; + + ret = sec_do_cipher_task(c_ctx->qp, &ecb_ctx); + if (ret) { + EMSG("XTS iv enc failed . ret = 0x%" PRIx32, ret); + return ret; + } + + for (i = 0; i < multiple_round(c_ctx->len, AES_SM4_BLOCK_SIZE); i++) + xts_multi_galois(c_ctx->iv); + + ecb_ctx.encrypt = false; + ret = sec_do_cipher_task(c_ctx->qp, &ecb_ctx); + if (ret) + EMSG("XTS iv denc failed . ret = 0x%" PRIx32, ret); + + return ret; +} + +static TEE_Result sec_update_iv(struct sec_cipher_ctx *c_ctx) +{ + TEE_Result ret = TEE_SUCCESS; + size_t offset = 0; + + switch (c_ctx->mode) { + case C_MODE_CBC: + offset = c_ctx->len - c_ctx->iv_len; + if (c_ctx->encrypt && c_ctx->len >= c_ctx->iv_len) + memcpy(c_ctx->iv, c_ctx->out + offset, c_ctx->iv_len); + if (!c_ctx->encrypt && c_ctx->len >= c_ctx->iv_len) + memcpy(c_ctx->iv, c_ctx->in + offset, c_ctx->iv_len); + break; + case C_MODE_CTR: + ctr_iv_inc(c_ctx->iv, c_ctx->len >> CTR_MODE_LEN_SHIFT); + break; + case C_MODE_XTS: + ret = xts_iv_update(c_ctx); + break; + default: + break; + } + + return ret; +} + +static TEE_Result sec_cipher_des_get_c_key_len(const int key_len, + uint8_t *c_key_len) +{ + if (key_len == DES_KEY_SIZE) { + *c_key_len = CKEY_LEN_DES; + } else { + EMSG("Invalid DES key size"); + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static TEE_Result sec_cipher_3des_get_c_key_len(const int key_len, + uint8_t *c_key_len) +{ + if (key_len == SEC_3DES_2KEY_SIZE) { + *c_key_len = CKEY_LEN_3DES_2KEY; + } else if (key_len == SEC_3DES_3KEY_SIZE) { + *c_key_len = CKEY_LEN_3DES_3KEY; + } else { + EMSG("Invalid 3DES key size"); + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static TEE_Result sec_cipher_aes_get_c_key_len(const int key_len, + const uint8_t mode, + uint8_t *c_key_len) +{ + switch (mode) { + case C_MODE_ECB: + case C_MODE_CBC: + case C_MODE_CTR: + case C_MODE_GCM: + switch (key_len) { + case AES_KEYSIZE_128: + *c_key_len = CKEY_LEN_128_BIT; + break; + case AES_KEYSIZE_192: + *c_key_len = CKEY_LEN_192_BIT; + break; + case AES_KEYSIZE_256: + *c_key_len = CKEY_LEN_256_BIT; + break; + default: + EMSG("Invalid AES key size"); + return TEE_ERROR_BAD_PARAMETERS; + } + break; + case C_MODE_XTS: + switch (key_len) { + case XTS_KEYSIZE_128: + *c_key_len = CKEY_LEN_128_BIT; + break; + case XTS_KEYSIZE_256: + *c_key_len = CKEY_LEN_256_BIT; + break; + default: + EMSG("Invalid AES-XTS key size"); + return TEE_ERROR_BAD_PARAMETERS; + } + break; + default: + EMSG("Unsupported AES mode\n"); + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static TEE_Result sec_cipher_sm4_get_c_key_len(const int key_len, + const uint8_t mode, + uint8_t *c_key_len) +{ + switch (mode) { + case C_MODE_ECB: + case C_MODE_CBC: + case C_MODE_CTR: + if (key_len != AES_KEYSIZE_128) { + EMSG("Invalid SM4 key size"); + return TEE_ERROR_BAD_PARAMETERS; + } + *c_key_len = CKEY_LEN_128_BIT; + break; + case C_MODE_XTS: + if (key_len != XTS_KEYSIZE_128) { + EMSG("Invalid SM4-XTS key size"); + return TEE_ERROR_BAD_PARAMETERS; + } + *c_key_len = CKEY_LEN_128_BIT; + break; + default: + EMSG("Unsupported SM4 mode\n"); + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static TEE_Result sec_cipher_set_key(struct sec_cipher_ctx *c_ctx, + const uint8_t *key1, const int key1_len, + const uint8_t *key2, const int key2_len) +{ + int key_len = key1_len + key2_len; + TEE_Result ret = TEE_SUCCESS; + uint8_t c_key_len = 0; + + switch (c_ctx->alg) { + case C_ALG_DES: + ret = sec_cipher_des_get_c_key_len(key_len, &c_key_len); + break; + case C_ALG_3DES: + ret = sec_cipher_3des_get_c_key_len(key_len, &c_key_len); + break; + case C_ALG_AES: + ret = sec_cipher_aes_get_c_key_len(key_len, + c_ctx->mode, &c_key_len); + break; + case C_ALG_SM4: + ret = sec_cipher_sm4_get_c_key_len(key_len, + c_ctx->mode, &c_key_len); + break; + default: + EMSG("Invalid cipher type 0x%" PRIx32, c_ctx->alg); + ret = TEE_ERROR_NOT_IMPLEMENTED; + break; + } + + if (ret) + return ret; + + c_ctx->key_dma = virt_to_phys(c_ctx->key); + c_ctx->key_len = key_len; + c_ctx->c_key_len = c_key_len; + + memcpy(c_ctx->key, key1, key1_len); + memcpy(c_ctx->key + key1_len, key2, key2_len); + + return ret; +} + +static TEE_Result sec_cipher_iv_check(struct sec_cipher_ctx *c_ctx, + const int iv_size) +{ + TEE_Result ret = TEE_SUCCESS; + + switch (c_ctx->mode) { + case C_MODE_ECB: + ret = iv_size == TEE_SUCCESS ? + TEE_SUCCESS : TEE_ERROR_BAD_PARAMETERS; + break; + case C_MODE_CBC: + if (c_ctx->alg == C_ALG_DES || c_ctx->alg == C_ALG_3DES) { + ret = iv_size == DES_CBC_IV_SIZE ? + TEE_SUCCESS : TEE_ERROR_BAD_PARAMETERS; + break; + } + fallthrough; + case C_MODE_XTS: + case C_MODE_CTR: + if (c_ctx->alg == C_ALG_AES || c_ctx->alg == C_ALG_SM4) { + ret = iv_size == AES_SM4_IV_SIZE ? + TEE_SUCCESS : TEE_ERROR_BAD_PARAMETERS; + break; + } + fallthrough; + default: + ret = TEE_ERROR_BAD_PARAMETERS; + break; + } + + if (ret) + EMSG("iv_size check failed"); + + return ret; +} + +static TEE_Result sec_cipher_set_iv(struct sec_cipher_ctx *c_ctx, + const uint8_t *iv, const int iv_len) +{ + TEE_Result ret = TEE_SUCCESS; + + if (!iv && iv_len) { + EMSG("IV is NULL"); + return TEE_ERROR_BAD_PARAMETERS; + } + + ret = sec_cipher_iv_check(c_ctx, iv_len); + if (ret) + return ret; + + c_ctx->iv_len = iv_len; + c_ctx->iv_dma = virt_to_phys(c_ctx->iv); + + memcpy(c_ctx->iv, iv, c_ctx->iv_len); + + return TEE_SUCCESS; +} + +static enum hisi_drv_status sec_cipher_bd_fill(void *bd, void *msg) +{ + struct sec_cipher_ctx *c_ctx = (struct sec_cipher_ctx *)msg; + struct hisi_sec_sqe *sqe = (struct hisi_sec_sqe *)bd; + + sqe->type = BD_TYPE2; + sqe->scene = SCENE_IPSEC; + sqe->de = DATA_DST_ADDR_ENABLE; + sqe->type2.c_len = c_ctx->len; + sqe->src_addr_type = HISI_FLAT_BUF; + sqe->dst_addr_type = HISI_FLAT_BUF; + sqe->type2.c_alg = c_ctx->alg; + sqe->type2.c_mode = c_ctx->mode; + sqe->type2.c_key_len = c_ctx->c_key_len; + + if (c_ctx->encrypt) + sqe->cipher = CIPHER_ENCRYPT; + else + sqe->cipher = CIPHER_DECRYPT; + + reg_pair_from_64(c_ctx->out_dma, &sqe->type2.data_dst_addr_h, + &sqe->type2.data_dst_addr_l); + reg_pair_from_64(c_ctx->in_dma, &sqe->type2.data_src_addr_h, + &sqe->type2.data_src_addr_l); + reg_pair_from_64(c_ctx->key_dma, &sqe->type2.c_key_addr_h, + &sqe->type2.c_key_addr_l); + if (c_ctx->iv_len == 0) + return HISI_QM_DRVCRYPT_NO_ERR; + + reg_pair_from_64(c_ctx->iv_dma, &sqe->type2.c_ivin_addr_h, + &sqe->type2.c_ivin_addr_l); + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static enum hisi_drv_status sec_cipher_bd_parse(void *bd, void *msg __unused) +{ + struct hisi_sec_sqe *sqe = (struct hisi_sec_sqe *)bd; + + if (sqe->type2.done != SEC_HW_TASK_DONE || sqe->type2.error_type) { + EMSG("SEC BD2 fail done=0x%" PRIx32 ", etype=0x%" PRIx32, + sqe->type2.done, sqe->type2.error_type); + return HISI_QM_DRVCRYPT_EINVAL; + } + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static enum hisi_drv_status sec_cipher_bd3_fill(void *bd, void *msg) +{ + struct hisi_sec_bd3_sqe *sqe = (struct hisi_sec_bd3_sqe *)bd; + struct sec_cipher_ctx *c_ctx = (struct sec_cipher_ctx *)msg; + + sqe->type = BD_TYPE3; + sqe->scene = SCENE_IPSEC; + sqe->de = DATA_DST_ADDR_ENABLE; + sqe->c_len = c_ctx->len; + sqe->src_addr_type = HISI_FLAT_BUF; + sqe->dst_addr_type = HISI_FLAT_BUF; + sqe->c_alg = c_ctx->alg; + sqe->c_mode = c_ctx->mode; + sqe->c_key_len = c_ctx->c_key_len; + + if (c_ctx->encrypt) + sqe->cipher = CIPHER_ENCRYPT; + else + sqe->cipher = CIPHER_DECRYPT; + + reg_pair_from_64(c_ctx->out_dma, &sqe->data_dst_addr_h, + &sqe->data_dst_addr_l); + reg_pair_from_64(c_ctx->in_dma, &sqe->data_src_addr_h, + &sqe->data_src_addr_l); + reg_pair_from_64(c_ctx->key_dma, &sqe->c_key_addr_h, + &sqe->c_key_addr_l); + + if (c_ctx->iv_len == 0) + return HISI_QM_DRVCRYPT_NO_ERR; + + reg_pair_from_64(c_ctx->iv_dma, &sqe->ipsec_scene.c_ivin_addr_h, + &sqe->ipsec_scene.c_ivin_addr_l); + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static enum hisi_drv_status sec_cipher_bd3_parse(void *bd, void *msg __unused) +{ + struct hisi_sec_bd3_sqe *sqe = (struct hisi_sec_bd3_sqe *)bd; + + if (sqe->done != SEC_HW_TASK_DONE || sqe->error_type) { + EMSG("SEC BD3 fail done=0x%" PRIx32 ", etype=0x%" PRIx32, + sqe->done, sqe->error_type); + return HISI_QM_DRVCRYPT_EINVAL; + } + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static TEE_Result cipher_algo_check(uint32_t algo) +{ + switch (algo) { + case TEE_ALG_AES_ECB_NOPAD: + case TEE_ALG_AES_CBC_NOPAD: + case TEE_ALG_AES_CTR: + case TEE_ALG_AES_XTS: + case TEE_ALG_DES_ECB_NOPAD: + case TEE_ALG_DES3_ECB_NOPAD: + case TEE_ALG_DES_CBC_NOPAD: + case TEE_ALG_DES3_CBC_NOPAD: + case TEE_ALG_SM4_CBC_NOPAD: + case TEE_ALG_SM4_ECB_NOPAD: + case TEE_ALG_SM4_CTR: + return TEE_SUCCESS; + default: + return TEE_ERROR_NOT_IMPLEMENTED; + } +} + +static TEE_Result crypto_set_calg(struct sec_cipher_ctx *c_ctx, + const uint32_t alg) +{ + TEE_Result ret = TEE_SUCCESS; + + switch (alg) { + case TEE_MAIN_ALGO_DES: + c_ctx->alg = C_ALG_DES; + break; + case TEE_MAIN_ALGO_DES3: + c_ctx->alg = C_ALG_3DES; + break; + case TEE_MAIN_ALGO_AES: + c_ctx->alg = C_ALG_AES; + break; + case TEE_MAIN_ALGO_SM4: + c_ctx->alg = C_ALG_SM4; + break; + default: + EMSG("Invalid cipher type 0x%"PRIx32, alg); + ret = TEE_ERROR_NOT_IMPLEMENTED; + break; + } + + return ret; +} + +static TEE_Result crypto_set_cmode(struct sec_cipher_ctx *c_ctx, + const uint32_t mode) +{ + TEE_Result ret = TEE_SUCCESS; + + switch (mode) { + case TEE_CHAIN_MODE_ECB_NOPAD: + c_ctx->mode = C_MODE_ECB; + break; + case TEE_CHAIN_MODE_CBC_NOPAD: + c_ctx->mode = C_MODE_CBC; + break; + case TEE_CHAIN_MODE_XTS: + c_ctx->mode = C_MODE_XTS; + break; + case TEE_CHAIN_MODE_CTR: + c_ctx->mode = C_MODE_CTR; + break; + default: + EMSG("Invalid cipher mode type 0x%" PRIx32, mode); + ret = TEE_ERROR_NOT_IMPLEMENTED; + break; + } + + return ret; +} + +static TEE_Result sec_cipher_alloc_ctx(void **ctx, uint32_t algo) +{ + struct sec_cipher_ctx *c_ctx = NULL; + TEE_Result ret = TEE_SUCCESS; + + if (!ctx) { + EMSG("Ctx is NULL"); + return TEE_ERROR_STORAGE_NO_SPACE; + } + + ret = cipher_algo_check(algo); + if (ret) + return ret; + + c_ctx = (struct sec_cipher_ctx *)malloc(sizeof(struct sec_cipher_ctx)); + if (!c_ctx) { + EMSG("c_ctx is NULL"); + return TEE_ERROR_STORAGE_NO_SPACE; + } + + ret = crypto_set_calg(c_ctx, TEE_ALG_GET_MAIN_ALG(algo)); + if (ret) + goto free_c_ctx; + + ret = crypto_set_cmode(c_ctx, TEE_ALG_GET_CHAIN_MODE(algo)); + if (ret) + goto free_c_ctx; + + c_ctx->qp = hisi_sec_create_qp(HISI_QM_CHANNEL_TYPE0); + if (!c_ctx->qp) { + ret = TEE_ERROR_BUSY; + goto free_c_ctx; + } + + if (c_ctx->qp->qm->version == HISI_QM_HW_V2) { + c_ctx->qp->fill_sqe = sec_cipher_bd_fill; + c_ctx->qp->parse_sqe = sec_cipher_bd_parse; + } else { + c_ctx->qp->fill_sqe = sec_cipher_bd3_fill; + c_ctx->qp->parse_sqe = sec_cipher_bd3_parse; + } + + c_ctx->offs = 0; + *ctx = c_ctx; + + return TEE_SUCCESS; + +free_c_ctx: + free(c_ctx); + + return ret; +} + +static void sec_cipher_free_ctx(void *ctx) +{ + struct sec_cipher_ctx *c_ctx = (struct sec_cipher_ctx *)ctx; + + if (!c_ctx) + return; + + hisi_qm_release_qp(c_ctx->qp); + memzero_explicit(c_ctx->key, c_ctx->key_len); + free(c_ctx); +} + +static TEE_Result sec_cipher_init(struct drvcrypt_cipher_init *dinit) +{ + struct sec_cipher_ctx *c_ctx = NULL; + TEE_Result ret = TEE_SUCCESS; + + if (!dinit || !dinit->ctx || !dinit->key1.data) { + EMSG("drvcrypt_cipher init param error"); + return TEE_ERROR_BAD_PARAMETERS; + } + + c_ctx = (struct sec_cipher_ctx *)dinit->ctx; + + ret = sec_cipher_set_key(c_ctx, dinit->key1.data, dinit->key1.length, + dinit->key2.data, dinit->key2.length); + if (ret) + return ret; + + ret = sec_cipher_set_iv(c_ctx, dinit->iv.data, dinit->iv.length); + if (ret) + return ret; + + c_ctx->encrypt = dinit->encrypt; + + return TEE_SUCCESS; +} + +static TEE_Result sec_cipher_cryptlen_check(struct sec_cipher_ctx *c_ctx, + size_t length) +{ + if (c_ctx->mode == C_MODE_XTS && length < AES_SM4_BLOCK_SIZE) { + EMSG("Invalid src len"); + return TEE_ERROR_BAD_PARAMETERS; + } + if ((c_ctx->mode == C_MODE_ECB || c_ctx->mode == C_MODE_CBC) && + (length & (AES_SM4_BLOCK_SIZE - 1))) { + EMSG("Invalid src len"); + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static TEE_Result sec_cipher_param_check(struct drvcrypt_cipher_update *dupdate) +{ + struct sec_cipher_ctx *c_ctx = NULL; + + if (!dupdate || !dupdate->src.data || !dupdate->dst.data || + dupdate->src.length != dupdate->dst.length || + dupdate->src.length > MAX_CIPHER_LENGTH || + !dupdate->src.length) { + EMSG("Dupdate input param error"); + return TEE_ERROR_BAD_PARAMETERS; + } + + c_ctx = (struct sec_cipher_ctx *)dupdate->ctx; + switch (c_ctx->alg) { + case C_ALG_SM4: + case C_ALG_AES: + if (sec_cipher_cryptlen_check(c_ctx, dupdate->src.length)) + return TEE_ERROR_BAD_PARAMETERS; + break; + case C_ALG_DES: + case C_ALG_3DES: + if (dupdate->src.length % TEE_DES_BLOCK_SIZE) { + EMSG("Invalid src len"); + return TEE_ERROR_BAD_PARAMETERS; + } + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static TEE_Result sec_alloc_buffer(struct sec_cipher_ctx *c_ctx) +{ + c_ctx->in = (uint8_t *)malloc(c_ctx->len); + if (!c_ctx->in) { + EMSG("Failed to alloc c_in buf"); + return TEE_ERROR_STORAGE_NO_SPACE; + } + + c_ctx->in_dma = virt_to_phys(c_ctx->in); + + c_ctx->out = (uint8_t *)malloc(c_ctx->len); + if (!c_ctx->out) { + EMSG("Failed to alloc c_out buf"); + goto free_c_in; + } + + c_ctx->out_dma = virt_to_phys(c_ctx->out); + + return TEE_SUCCESS; + +free_c_in: + free(c_ctx->in); + return TEE_ERROR_STORAGE_NO_SPACE; +} + +static void sec_free_buffer(struct sec_cipher_ctx *c_ctx) +{ + free(c_ctx->in); + free(c_ctx->out); +} + +static TEE_Result sec_cipher_update(struct drvcrypt_cipher_update *dupdate) +{ + struct sec_cipher_ctx *c_ctx = NULL; + size_t padding_size = 0; + TEE_Result ret = TEE_SUCCESS; + + ret = sec_cipher_param_check(dupdate); + if (ret) + return ret; + + c_ctx = (struct sec_cipher_ctx *)dupdate->ctx; + if (c_ctx->mode == C_MODE_CTR && (c_ctx->offs & CTR_SRC_ALIGN_MASK)) + padding_size = (c_ctx->offs % CTR_SRC_BLOCK_SIZE); + + c_ctx->offs += dupdate->src.length; + c_ctx->len = dupdate->src.length + padding_size; + ret = sec_alloc_buffer(c_ctx); + if (ret) + return ret; + + memzero_explicit(c_ctx->in, padding_size); + memcpy(c_ctx->in + padding_size, + dupdate->src.data, dupdate->src.length); + + ret = sec_do_cipher_task(c_ctx->qp, c_ctx); + if (ret) + goto free_buffer; + + ret = sec_update_iv(c_ctx); + if (ret) { + EMSG("sec_update_iv failed. ret = 0x%" PRIx32, ret); + goto free_buffer; + } + + memcpy(dupdate->dst.data, + c_ctx->out + padding_size, dupdate->src.length); + +free_buffer: + sec_free_buffer(c_ctx); + return ret; +} + +/* + * Finalize of the cipher operation + * + * @ctx Caller context variable or NULL + */ +static void sec_cipher_final(void *ctx __unused) +{ +} + +static void sec_cipher_copy_state(void *dst_ctx, void *src_ctx) +{ + struct sec_cipher_ctx *dst_c_ctx = (struct sec_cipher_ctx *)dst_ctx; + struct sec_cipher_ctx *src_c_ctx = (struct sec_cipher_ctx *)src_ctx; + + dst_c_ctx->alg = src_c_ctx->alg; + dst_c_ctx->mode = src_c_ctx->mode; + dst_c_ctx->encrypt = src_c_ctx->encrypt; + dst_c_ctx->offs = src_c_ctx->offs; + + if (src_c_ctx->key_len) { + dst_c_ctx->key_len = src_c_ctx->key_len; + dst_c_ctx->c_key_len = src_c_ctx->c_key_len; + memcpy(dst_c_ctx->key, src_c_ctx->key, dst_c_ctx->key_len); + dst_c_ctx->key_dma = virt_to_phys(dst_c_ctx->key); + } + + if (src_c_ctx->iv_len) { + dst_c_ctx->iv_len = src_c_ctx->iv_len; + memcpy(dst_c_ctx->iv, src_c_ctx->iv, dst_c_ctx->iv_len); + dst_c_ctx->iv_dma = virt_to_phys(dst_c_ctx->iv); + } +} + +/* + * Registration of the Cipher Driver + */ +static struct drvcrypt_cipher driver_cipher = { + .alloc_ctx = sec_cipher_alloc_ctx, + .free_ctx = sec_cipher_free_ctx, + .init = sec_cipher_init, + .update = sec_cipher_update, + .final = sec_cipher_final, + .copy_state = sec_cipher_copy_state, +}; + +/* + * Initialize the Cipher module + * + */ +static TEE_Result sec_init_cipher(void) +{ + TEE_Result ret = drvcrypt_register_cipher(&driver_cipher); + + if (ret) + EMSG("Sec cipher register failed. ret = 0x%" PRIx32, ret); + + return ret; +} +driver_init(sec_init_cipher); diff --git a/core/drivers/crypto/hisilicon/sec/sub.mk b/core/drivers/crypto/hisilicon/sec/sub.mk index 2f3e044ad41..9cef92b9837 100644 --- a/core/drivers/crypto/hisilicon/sec/sub.mk +++ b/core/drivers/crypto/hisilicon/sec/sub.mk @@ -1,3 +1,4 @@ incdirs-y += ../include srcs-y += hisi_sec.c +srcs-y += sec_cipher.c