diff --git a/core/arch/arm/plat-d06/conf.mk b/core/arch/arm/plat-d06/conf.mk index 98a8fa01bbe..b0830863dc7 100644 --- a/core/arch/arm/plat-d06/conf.mk +++ b/core/arch/arm/plat-d06/conf.mk @@ -8,7 +8,9 @@ CFG_WITH_PAGER ?= n CFG_WITH_SOFTWARE_PRNG ?= y CFG_WITH_STATS ?= y CFG_TEE_CORE_EMBED_INTERNAL_TESTS ?= y +CFG_HISILICON_CRYPTO_DRIVER ?= y +$(call force,CFG_GIC,y) $(call force,CFG_SECURE_TIME_SOURCE_CNTPCT,y) $(call force,CFG_WITH_ARM_TRUSTED_FW,y) $(call force,CFG_ARM64_core,y) diff --git a/core/arch/arm/plat-d06/main.c b/core/arch/arm/plat-d06/main.c index 77369440117..8c36bdd0214 100644 --- a/core/arch/arm/plat-d06/main.c +++ b/core/arch/arm/plat-d06/main.c @@ -10,6 +10,7 @@ static struct lpc_uart_data console_data __nex_bss; register_phys_mem_pgdir(MEM_AREA_IO_NSEC, LPC_BASE, LPC_SIZE); +register_phys_mem_pgdir(MEM_AREA_IO_SEC, SEC_BASE, SEC_SIZE); void console_init(void) { diff --git a/core/arch/arm/plat-d06/platform_config.h b/core/arch/arm/plat-d06/platform_config.h index 81979bdc478..de82a802f98 100644 --- a/core/arch/arm/plat-d06/platform_config.h +++ b/core/arch/arm/plat-d06/platform_config.h @@ -17,4 +17,8 @@ #define CONSOLE_BAUDRATE 115200 #define CONSOLE_UART_CLK_IN_HZ 200 +/* SEC */ +#define SEC_BASE 0x141800000 +#define SEC_SIZE 0x400000 + #endif /* PLATFORM_CONFIG_H */ diff --git a/core/drivers/crypto/hisilicon/crypto.mk b/core/drivers/crypto/hisilicon/crypto.mk new file mode 100644 index 00000000000..746f74c6511 --- /dev/null +++ b/core/drivers/crypto/hisilicon/crypto.mk @@ -0,0 +1,6 @@ +ifeq ($(CFG_HISILICON_CRYPTO_DRIVER),y) +$(call force,CFG_CRYPTO_DRIVER,y) +CFG_CRYPTO_DRIVER_DEBUG ?= 0 +$(call force,CFG_CRYPTO_DRV_CIPHER,y,Mandated by CFG_HISILICON_CRYPTO_DRIVER) +$(call force,CFG_CRYPTO_DRV_ACIPHER,y,Mandated by CFG_HISILICON_CRYPTO_DRIVER) +endif diff --git a/core/drivers/crypto/hisilicon/hisi_qm.c b/core/drivers/crypto/hisilicon/hisi_qm.c new file mode 100644 index 00000000000..6c1e5960e0c --- /dev/null +++ b/core/drivers/crypto/hisilicon/hisi_qm.c @@ -0,0 +1,786 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 HiSilicon Limited. + * Kunpeng hardware accelerator queue management module. + */ +#include "hisi_qm.h" + +#define QM_FVT_CFG_RDY_BIT 0x1 +/* doorbell */ +#define QM_DOORBELL_SQ_CQ_BASE 0x1000 +#define QM_DB_CMD_SHIFT 12 +#define QM_DB_RAND_DATA_SHIFT 16 +#define QM_DB_INDEX_SHIFT 32 +#define QM_DB_PRIORITY_SHIFT 48 +#define QM_DB_RAND_DATA 0x5a +#define QM_DOORBELL_CMD_SQ 0 +#define QM_DOORBELL_CMD_CQ 1 +/* mailbox */ +#define QM_MAILBOX_BASE 0x300 +#define QM_MAILBOX_DATA_ADDR_L 0x304 +#define QM_MAILBOX_DATA_ADDR_H 0x308 +#define QM_MB_BUSY_SHIFT 13 +#define QM_MB_BUSY_BIT BIT32(QM_MB_BUSY_SHIFT) +#define QM_MB_OP_SHIFT 14 +#define QM_MB_OP_WR 0 +#define QM_MB_OP_RD 1 +/* XQC_VFT */ +#define QM_VFT_CFG_OP_ENABLE 0x100054 +#define QM_VFT_CFG_OP_WR 0x100058 +#define QM_VFT_CFG_TYPE 0x10005c +#define QM_VFT_CFG_ADDRESS 0x100060 +#define QM_VFT_CFG_DATA_L 0x100064 +#define QM_VFT_CFG_DATA_H 0x100068 +#define QM_VFT_CFG_RDY 0x10006c +#define QM_SQC_VFT 0 +#define QM_CQC_VFT 1 +#define QM_SQC_VFT_START_SQN_SHIFT 28 +#define QM_SQC_VFT_VALID BIT64(44) +#define QM_SQC_VFT_SQ_NUM_SHIFT 45 +#define QM_CQC_VFT_VALID BIT(28) +#define QM_VFT_WRITE 0 +#define QM_VFT_READ 1 +#define QM_SQC_VFT_BASE_MASK 0x3ff +#define QM_SQC_VFT_NUM_MASK 0x3ff +/* QM INIT */ +#define QM_MEM_START_INIT 0x100040 +#define QM_MEM_INIT_DONE 0x100044 +#define QM_VF_AEQ_INT_MASK 0x4 +#define QM_VF_AEQ_INT_MASK_EN 0x1 +#define QM_VF_EQ_INT_MASK 0xc +#define QM_VF_EQ_INT_MASK_EN 0x1 +#define QM_ARUSER_M_CFG_1 0x100088 +#define QM_ARUSER_M_CFG_ENABLE 0x100090 +#define QM_AWUSER_M_CFG_1 0x100098 +#define QM_AWUSER_M_CFG_ENABLE 0x1000a0 +#define QM_AXUSER_CFG 0x40001070 +#define AXUSER_M_CFG_ENABLE 0x7ffffc +#define QM_AXI_M_CFG 0x1000ac +#define AXI_M_CFG 0xffff +#define QM_PEH_AXUSER_CFG 0x1000cc +#define PEH_AXUSER_CFG 0x400801 +#define QM_CACHE_CTL 0x100050 +#define QM_CACHE_CFG 0x4893 +#define QM_CACHE_WB_START 0x100204 +#define QM_CACHE_WB_DONE 0x100208 +/* XQC shift */ +#define QM_SQ_SQE_SIZE_SHIFT 12 +#define QM_SQ_ORDER_SHIFT 4 +#define QM_SQ_TYPE_SHIFT 8 +#define QM_CQE_SIZE 4 +#define QM_CQ_CQE_SIZE_SHIFT 12 +/* CQE */ +#define QM_CQE_PHASE(cqe) (((cqe)->w7) & QM_FVT_CFG_RDY_BIT) + +enum qm_mailbox_common_cmd { + QM_MB_CMD_SQC = 0x0, + QM_MB_CMD_CQC, + QM_MB_CMD_EQC, + QM_MB_CMD_AEQC, + QM_MB_CMD_SQC_BT, + QM_MB_CMD_CQC_BT, + QM_MB_CMD_SQC_VFT, +}; + +enum qm_mailbox_cmd_v3 { + QM_MB_CM_CLOSE_QM = 0x7, + QM_MB_CMD_CLOSE_QP, + QM_MB_CMD_FLUSH_QM, + QM_MB_CMD_FLUSH_QP, + QM_MB_CMD_SRC = 0xc, + QM_MB_CMD_DST, + QM_MB_CMD_STOP_QM, +}; + +struct qm_mailbox { + uint16_t w0; + uint16_t queue; + uint32_t base_l; + uint32_t base_h; + uint32_t token; +}; + +struct qm_dfx_registers { + const char *reg_name; + uint32_t reg_offset; +}; + +static const struct qm_dfx_registers qm_dfx_regs[] = { + { .reg_name = "QM_ECC_1BIT_CNT ", .reg_offset = 0x104000 }, + { .reg_name = "QM_ECC_MBIT_CNT ", .reg_offset = 0x104008 }, + { .reg_name = "QM_DFX_MB_CNT ", .reg_offset = 0x104018 }, + { .reg_name = "QM_DFX_DB_CNT ", .reg_offset = 0x104028 }, + { .reg_name = "QM_DFX_SQE_CNT ", .reg_offset = 0x104038 }, + { .reg_name = "QM_DFX_CQE_CNT ", .reg_offset = 0x104048 }, + { .reg_name = "QM_DFX_SEND_SQE_TO_ACC_CNT", .reg_offset = 0x104050 }, + { .reg_name = "QM_DFX_WB_SQE_FROM_ACC_CNT", .reg_offset = 0x104058 }, + { .reg_name = "QM_DFX_ACC_FINISH_CNT ", .reg_offset = 0x104060 }, + { .reg_name = "QM_DFX_CQE_ERR_CNT ", .reg_offset = 0x1040b4 }, + { .reg_name = NULL, 0 } +}; + +void hisi_qm_get_version(struct hisi_qm *qm) +{ + qm->version = io_read32(qm->io_base + HISI_QM_REVISON_ID_BASE) & + HISI_QM_REVISON_ID_MASK; +} + +static void qm_db(struct hisi_qm *qm, uint16_t qn, uint8_t cmd, uint16_t index, + uint8_t priority) +{ + uint64_t doorbell = 0; + + doorbell = qn | SHIFT_U64(cmd, QM_DB_CMD_SHIFT) | + SHIFT_U64(QM_DB_RAND_DATA, QM_DB_RAND_DATA_SHIFT) | + SHIFT_U64(index, QM_DB_INDEX_SHIFT) | + SHIFT_U64(priority, QM_DB_PRIORITY_SHIFT); + + io_write64(qm->io_base + QM_DOORBELL_SQ_CQ_BASE, doorbell); +} + +static enum hisi_drv_status qm_wait_mb_ready(struct hisi_qm *qm) +{ + uint32_t val = 0; + + /* return 0 mailbox ready, HISI_QM_DRVCRYPT_ETMOUT hardware timeout */ + return readl_relaxed_poll_timeout(qm->io_base + QM_MAILBOX_BASE, val, + !(val & QM_MB_BUSY_BIT), POLL_PERIOD, + POLL_TIMEOUT); +} + +static void qm_mb_write(struct hisi_qm *qm, void *src) +{ + vaddr_t dst = qm->io_base + QM_MAILBOX_BASE; + unsigned long tmp0 = 0; + unsigned long tmp1 = 0; + + /* 128bits should be written to hardware at one time */ + asm volatile ("ldp %0, %1, %3\n" + "stp %0, %1, %2\n" + : "=&r"(tmp0), "=&r"(tmp1), "+Q"(*((char *)dst)) + : "Q"(*((char *)src)) + : "memory"); + dsb(); +} + +static enum hisi_drv_status qm_mb(struct hisi_qm *qm, uint8_t cmd, + vaddr_t dma_addr, uint16_t qn, uint8_t op) +{ + struct qm_mailbox mb = { }; + + mb.w0 = cmd | SHIFT_U32(op, QM_MB_OP_SHIFT) | + BIT32(QM_MB_BUSY_SHIFT); + mb.queue = qn; + reg_pair_from_64(dma_addr, &mb.base_h, &mb.base_l); + mb.token = 0; + + if (qm_wait_mb_ready(qm)) { + EMSG("QM mailbox is busy"); + return HISI_QM_DRVCRYPT_EBUSY; + } + + qm_mb_write(qm, &mb); + + if (qm_wait_mb_ready(qm)) { + EMSG("QM mailbox operation timeout"); + return HISI_QM_DRVCRYPT_EBUSY; + } + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static void qm_cfg_vft_data(struct hisi_qm *qm, uint8_t vft_type, + uint32_t base, uint32_t number) +{ + uint32_t data_h = 0; + uint32_t data_l = 0; + uint64_t data = 0; + + switch (vft_type) { + case QM_SQC_VFT: + data = SHIFT_U64(base, QM_SQC_VFT_START_SQN_SHIFT) | + QM_SQC_VFT_VALID | + SHIFT_U64((number - 1), QM_SQC_VFT_SQ_NUM_SHIFT); + break; + case QM_CQC_VFT: + data = QM_CQC_VFT_VALID; + break; + default: + panic("Invalid vft type"); + } + + reg_pair_from_64(data, &data_h, &data_l); + io_write32(qm->io_base + QM_VFT_CFG_DATA_L, data_l); + io_write32(qm->io_base + QM_VFT_CFG_DATA_H, data_h); +} + +static enum hisi_drv_status qm_set_vft_common(struct hisi_qm *qm, + uint8_t vft_type, + uint32_t function, + uint32_t base, + uint32_t num) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + uint32_t val = 0; + + ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, + val & QM_FVT_CFG_RDY_BIT, POLL_PERIOD, + POLL_TIMEOUT); + if (ret) { + EMSG("QM VFT is not ready"); + return ret; + } + + io_write32(qm->io_base + QM_VFT_CFG_OP_WR, QM_VFT_WRITE); + io_write32(qm->io_base + QM_VFT_CFG_TYPE, vft_type); + io_write32(qm->io_base + QM_VFT_CFG_ADDRESS, function); + qm_cfg_vft_data(qm, vft_type, base, num); + io_write32(qm->io_base + QM_VFT_CFG_RDY, 0x0); + io_write32(qm->io_base + QM_VFT_CFG_OP_ENABLE, QM_FVT_CFG_RDY_BIT); + + return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, + val & QM_FVT_CFG_RDY_BIT, POLL_PERIOD, + POLL_TIMEOUT); +} + +static enum hisi_drv_status qm_set_xqc_vft(struct hisi_qm *qm, + uint32_t function, + uint32_t base, uint32_t num) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + int32_t i = 0; + + if (!num) { + EMSG("Invalid sq num"); + return HISI_QM_DRVCRYPT_EINVAL; + } + + for (i = QM_SQC_VFT; i <= QM_CQC_VFT; i++) { + ret = qm_set_vft_common(qm, i, function, base, num); + if (ret) { + EMSG("QM set type%d fail", i); + return ret; + } + } + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static enum hisi_drv_status qm_get_vft(struct hisi_qm *qm, uint32_t *base, + uint32_t *num) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + uint64_t sqc_vft = 0; + + ret = qm_mb(qm, QM_MB_CMD_SQC_VFT, 0, 0, QM_MB_OP_RD); + if (ret) + return ret; + + sqc_vft = io_read64(qm->io_base + QM_MAILBOX_DATA_ADDR_L); + *base = (sqc_vft >> QM_SQC_VFT_START_SQN_SHIFT) & QM_SQC_VFT_BASE_MASK; + *num = ((sqc_vft >> QM_SQC_VFT_SQ_NUM_SHIFT) & QM_SQC_VFT_NUM_MASK) + 1; + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static void qp_memory_uninit(struct hisi_qm *qm, uint32_t id) +{ + struct hisi_qp *qp = &qm->qp_array[id]; + + free(qp->sqe); + free(qp->cqe); +} + +static enum hisi_drv_status qp_memory_init(struct hisi_qm *qm, uint32_t id) +{ + size_t sq_size = qm->sqe_size * HISI_QM_Q_DEPTH; + size_t cq_size = sizeof(struct qm_cqe) * HISI_QM_Q_DEPTH; + struct hisi_qp *qp = &qm->qp_array[id]; + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + + qp->sqe = memalign(HISI_QM_ALIGN128, sq_size); + if (!qp->sqe) { + EMSG("Fail to malloc sq[%"PRIu32"]", id); + return HISI_QM_DRVCRYPT_ENOMEM; + } + qp->sqe_dma = virt_to_phys(qp->sqe); + qp->cqe = memalign(HISI_QM_ALIGN32, cq_size); + if (!qp->cqe) { + EMSG("Fail to malloc cq[%"PRIu32"]", id); + ret = HISI_QM_DRVCRYPT_ENOMEM; + goto free_sqe; + } + qp->cqe_dma = virt_to_phys(qp->cqe); + + qp->qp_id = id; + qp->qm = qm; + return HISI_QM_DRVCRYPT_NO_ERR; + +free_sqe: + free(qp->sqe); + return ret; +} + +static void qm_memory_uninit(struct hisi_qm *qm) +{ + uint32_t i = 0; + + for (i = 0; i < qm->qp_num; i++) + qp_memory_uninit(qm, i); + + free(qm->qp_array); + free(qm->sqc); + free(qm->cqc); +} + +static enum hisi_drv_status qm_memory_init(struct hisi_qm *qm) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + size_t sqc_size = 0; + size_t cqc_size = 0; + size_t qp_size = 0; + uint32_t i = 0; + int32_t j = 0; + + sqc_size = sizeof(struct qm_sqc) * qm->qp_num; + cqc_size = sizeof(struct qm_cqc) * qm->qp_num; + qp_size = sizeof(struct hisi_qp) * qm->qp_num; + + qm->sqc = memalign(HISI_QM_ALIGN32, sqc_size); + if (!qm->sqc) { + EMSG("Fail to malloc sqc"); + return HISI_QM_DRVCRYPT_ENOMEM; + } + qm->sqc_dma = virt_to_phys(qm->sqc); + + qm->cqc = memalign(HISI_QM_ALIGN32, cqc_size); + if (!qm->cqc) { + EMSG("Fail to malloc cqc"); + ret = HISI_QM_DRVCRYPT_ENOMEM; + goto free_sqc; + } + qm->cqc_dma = virt_to_phys(qm->cqc); + + qm->qp_array = (struct hisi_qp *)malloc(qp_size); + if (!qm->qp_array) { + EMSG("Fail to malloc qp_array"); + ret = HISI_QM_DRVCRYPT_ENOMEM; + goto free_cqc; + } + + for (i = 0; i < qm->qp_num; i++) { + ret = qp_memory_init(qm, i); + if (ret) { + ret = HISI_QM_DRVCRYPT_ENOMEM; + goto free_qp_mem; + } + } + + return HISI_QM_DRVCRYPT_NO_ERR; + +free_qp_mem: + for (j = (int32_t)i - 1; j >= 0; j--) + qp_memory_uninit(qm, j); + free(qm->qp_array); +free_cqc: + free(qm->cqc); +free_sqc: + free(qm->sqc); + return ret; +} + +enum hisi_drv_status hisi_qm_init(struct hisi_qm *qm) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + + if (qm->fun_type == HISI_QM_HW_VF) { + ret = qm_get_vft(qm, &qm->qp_base, &qm->qp_num); + if (ret) { + EMSG("Fail to get function vft config"); + return ret; + } + } + + if (qm->qp_num == 0 || qm->sqe_size == 0) { + EMSG("Invalid qm parameters"); + return HISI_QM_DRVCRYPT_EINVAL; + } + + ret = qm_memory_init(qm); + if (ret) + return ret; + + qm->qp_in_used = 0; + qm->qp_idx = 0; + mutex_init(&qm->qp_lock); + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static void qm_cache_writeback(struct hisi_qm *qm) +{ + uint32_t val = 0; + + io_write32(qm->io_base + QM_CACHE_WB_START, QM_FVT_CFG_RDY_BIT); + + if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE, val, + val & QM_FVT_CFG_RDY_BIT, POLL_PERIOD, + POLL_TIMEOUT)) + panic("QM writeback sqc cache fail"); +} + +void hisi_qm_uninit(struct hisi_qm *qm) +{ + qm_cache_writeback(qm); + qm_memory_uninit(qm); + mutex_destroy(&qm->qp_lock); +} + +static enum hisi_drv_status qm_hw_mem_reset(struct hisi_qm *qm) +{ + uint32_t val = 0; + + io_write32(qm->io_base + QM_MEM_START_INIT, QM_FVT_CFG_RDY_BIT); + + return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val, + val & QM_FVT_CFG_RDY_BIT, POLL_PERIOD, + POLL_TIMEOUT); +} + +static enum hisi_drv_status qm_func_vft_cfg(struct hisi_qm *qm) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + uint32_t q_base = qm->qp_num; + uint32_t act_q_num = 0; + uint32_t i = 0; + uint32_t j = 0; + + if (qm->vfs_num == 0) + return HISI_QM_DRVCRYPT_NO_ERR; + + if (qm->vfs_num > HISI_QM_MAX_VFS_NUM) { + EMSG("Invalid QM vfs_num"); + return HISI_QM_DRVCRYPT_EINVAL; + } + + for (i = 1; i <= qm->vfs_num; i++) { + act_q_num = HISI_QM_VF_Q_NUM; + ret = qm_set_xqc_vft(qm, i, q_base, act_q_num); + if (ret) { + for (j = 1; j < i; j++) + (void)qm_set_xqc_vft(qm, j, 0, 0); + return ret; + } + q_base += act_q_num; + } + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +enum hisi_drv_status hisi_qm_start(struct hisi_qm *qm) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + + if (qm->fun_type == HISI_QM_HW_PF) { + ret = qm_hw_mem_reset(qm); + if (ret) { + EMSG("Fail to reset qm hardware mem"); + return ret; + } + + ret = qm_set_xqc_vft(qm, 0, qm->qp_base, qm->qp_num); + if (ret) { + EMSG("Fail to set PF xqc_vft"); + return ret; + } + + ret = qm_func_vft_cfg(qm); + if (ret) { + EMSG("Fail to set VF xqc_vft"); + return ret; + } + } + + ret = qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, QM_MB_OP_WR); + if (ret) { + EMSG("Fail to set sqc_bt"); + return ret; + } + + ret = qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, QM_MB_OP_WR); + if (ret) { + EMSG("Fail to set cqc_bt"); + return ret; + } + + /* security mode does not support msi */ + io_write32(qm->io_base + QM_VF_AEQ_INT_MASK, QM_VF_AEQ_INT_MASK_EN); + io_write32(qm->io_base + QM_VF_EQ_INT_MASK, QM_VF_EQ_INT_MASK_EN); + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +void hisi_qm_dev_init(struct hisi_qm *qm) +{ + if (qm->fun_type == HISI_QM_HW_VF) + return; + + /* qm user domain */ + io_write32(qm->io_base + QM_ARUSER_M_CFG_1, QM_AXUSER_CFG); + io_write32(qm->io_base + QM_ARUSER_M_CFG_ENABLE, AXUSER_M_CFG_ENABLE); + io_write32(qm->io_base + QM_AWUSER_M_CFG_1, QM_AXUSER_CFG); + io_write32(qm->io_base + QM_AWUSER_M_CFG_ENABLE, AXUSER_M_CFG_ENABLE); + /* qm cache */ + io_write32(qm->io_base + QM_AXI_M_CFG, AXI_M_CFG); + + if (qm->version == HISI_QM_HW_V2) { + /* disable FLR triggered by BME(bus master enable) */ + io_write32(qm->io_base + QM_PEH_AXUSER_CFG, PEH_AXUSER_CFG); + /* set sec sqc and cqc cache wb threshold 4 */ + io_write32(qm->io_base + QM_CACHE_CTL, QM_CACHE_CFG); + } + /* disable qm ras */ + io_write32(qm->io_base + HISI_QM_ABNML_INT_MASK, + HISI_QM_ABNML_INT_MASK_CFG); +} + +static enum hisi_drv_status qm_sqc_cfg(struct hisi_qp *qp) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + struct hisi_qm *qm = qp->qm; + struct qm_sqc *sqc = NULL; + paddr_t sqc_dma = 0; + + sqc = memalign(HISI_QM_ALIGN32, sizeof(struct qm_sqc)); + if (!sqc) + return HISI_QM_DRVCRYPT_ENOMEM; + + sqc_dma = virt_to_phys(sqc); + + memzero_explicit(sqc, sizeof(struct qm_sqc)); + reg_pair_from_64(qp->sqe_dma, &sqc->base_h, &sqc->base_l); + sqc->dw3 = (HISI_QM_Q_DEPTH - 1) | + SHIFT_U32(qm->sqe_log2_size, QM_SQ_SQE_SIZE_SHIFT); + sqc->rand_data = QM_DB_RAND_DATA; + sqc->cq_num = qp->qp_id; + sqc->w13 = BIT32(QM_SQ_ORDER_SHIFT) | + SHIFT_U32(qp->sq_type, QM_SQ_TYPE_SHIFT); + + ret = qm_mb(qm, QM_MB_CMD_SQC, sqc_dma, qp->qp_id, QM_MB_OP_WR); + free(sqc); + + return ret; +} + +static enum hisi_drv_status qm_cqc_cfg(struct hisi_qp *qp) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + struct hisi_qm *qm = qp->qm; + struct qm_cqc *cqc = NULL; + paddr_t cqc_dma = 0; + + cqc = memalign(HISI_QM_ALIGN32, sizeof(struct qm_cqc)); + if (!cqc) + return HISI_QM_DRVCRYPT_ENOMEM; + + cqc_dma = virt_to_phys(cqc); + + memzero_explicit(cqc, sizeof(struct qm_cqc)); + reg_pair_from_64(qp->cqe_dma, &cqc->base_h, &cqc->base_l); + cqc->dw3 = (HISI_QM_Q_DEPTH - 1) | + SHIFT_U32(QM_CQE_SIZE, QM_CQ_CQE_SIZE_SHIFT); + cqc->rand_data = QM_DB_RAND_DATA; + cqc->dw6 = PHASE_DEFAULT_VAL; + + ret = qm_mb(qm, QM_MB_CMD_CQC, cqc_dma, qp->qp_id, QM_MB_OP_WR); + free(cqc); + + return ret; +} + +struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, uint8_t sq_type) +{ + struct hisi_qp *qp = NULL; + + mutex_lock(&qm->qp_lock); + if (qm->qp_in_used == qm->qp_num) { + EMSG("All %"PRIu32" queues of QM are busy", qm->qp_num); + goto err_proc; + } + + if (qm->qp_idx == qm->qp_num - 1) + qm->qp_idx = 0; + else + qm->qp_idx++; + + qp = &qm->qp_array[qm->qp_idx]; + memzero_explicit(qp->cqe, sizeof(struct qm_cqe) * HISI_QM_Q_DEPTH); + qp->sq_type = sq_type; + qp->sq_tail = 0; + qp->cq_head = 0; + qp->cqc_phase = true; + + if (qm_sqc_cfg(qp)) { + EMSG("Fail to set qp[%"PRIu32"] sqc", qp->qp_id); + goto err_proc; + } + + if (qm_cqc_cfg(qp)) { + EMSG("Fail to set qp[%"PRIu32"] cqc", qp->qp_id); + goto err_proc; + } + + qm->qp_in_used++; + mutex_unlock(&qm->qp_lock); + return qp; + +err_proc: + mutex_unlock(&qm->qp_lock); + return NULL; +} + +void hisi_qm_release_qp(struct hisi_qp *qp) +{ + struct hisi_qm *qm = NULL; + + if (!qp) { + EMSG("qp is NULL"); + return; + } + + qm = qp->qm; + mutex_lock(&qm->qp_lock); + qm->qp_in_used--; + mutex_unlock(&qm->qp_lock); +} + +static void qm_sq_tail_update(struct hisi_qp *qp) +{ + if (qp->sq_tail == HISI_QM_Q_DEPTH - 1) + qp->sq_tail = 0; + else + qp->sq_tail++; +} + +/* + * One task thread will just bind to one hardware queue, and + * hardware does not support msi. So we have no lock here. + */ +enum hisi_drv_status hisi_qp_send(struct hisi_qp *qp, void *msg) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + struct hisi_qm *qm = NULL; + void *sqe = NULL; + + if (!qp) { + EMSG("qp is NULL"); + return HISI_QM_DRVCRYPT_EINVAL; + } + + qm = qp->qm; + ret = qm->dev_status_check(qm); + if (ret) + return ret; + + sqe = (void *)((vaddr_t)qp->sqe + qm->sqe_size * qp->sq_tail); + memzero_explicit(sqe, qm->sqe_size); + + ret = qp->fill_sqe(sqe, msg); + if (ret) { + EMSG("Fail to fill sqe"); + return ret; + } + + qm_sq_tail_update(qp); + + dsb(); + qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_SQ, qp->sq_tail, 0); + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static void qm_cq_head_update(struct hisi_qp *qp) +{ + if (qp->cq_head == HISI_QM_Q_DEPTH - 1) { + qp->cqc_phase = !qp->cqc_phase; + qp->cq_head = 0; + } else { + qp->cq_head++; + } +} + +#define HISI_QM_RECV_DONE 0xAF +static enum hisi_drv_status hisi_qp_recv(struct hisi_qp *qp, void *msg) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + struct hisi_qm *qm = qp->qm; + struct qm_cqe *cqe = NULL; + void *sqe = NULL; + + ret = qm->dev_status_check(qm); + if (ret) + return ret; + + cqe = qp->cqe + qp->cq_head; + if (QM_CQE_PHASE(cqe) == qp->cqc_phase) { + dsb_osh(); + sqe = (void *)((vaddr_t)qp->sqe + qm->sqe_size * cqe->sq_head); + ret = qp->parse_sqe(sqe, msg); + qm_cq_head_update(qp); + qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, qp->cq_head, 0); + if (ret) { + EMSG("Fail to parse sqe"); + return ret; + } + } else { + return HISI_QM_DRVCRYPT_NO_ERR; + } + + return HISI_QM_RECV_DONE; +} + +static void qm_dfx_dump(struct hisi_qm *qm) +{ + const struct qm_dfx_registers *regs = qm_dfx_regs; + __maybe_unused uint32_t val = 0; + + if (qm->fun_type == HISI_QM_HW_VF) + return; + + while (regs->reg_name) { + val = io_read32(qm->io_base + regs->reg_offset); + EMSG("%s= 0x%" PRIx32, regs->reg_name, val); + regs++; + } +} + +enum hisi_drv_status hisi_qp_recv_sync(struct hisi_qp *qp, void *msg) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + uint32_t cnt = 0; + + if (!qp) { + EMSG("qp is NULL"); + return HISI_QM_DRVCRYPT_EINVAL; + } + + while (true) { + ret = hisi_qp_recv(qp, msg); + if (ret == HISI_QM_DRVCRYPT_NO_ERR) { + if (++cnt > HISI_QM_RECV_SYNC_TIMEOUT) { + EMSG("qm recv task timeout"); + qm_dfx_dump(qp->qm); + return HISI_QM_DRVCRYPT_ETMOUT; + } + } else { + if (ret == HISI_QM_RECV_DONE) + return HISI_QM_DRVCRYPT_NO_ERR; + + EMSG("qm recv task error"); + qm_dfx_dump(qp->qm); + break; + } + } + + return ret; +} diff --git a/core/drivers/crypto/hisilicon/include/hisi_cipher.h b/core/drivers/crypto/hisilicon/include/hisi_cipher.h new file mode 100644 index 00000000000..1f31ceb8012 --- /dev/null +++ b/core/drivers/crypto/hisilicon/include/hisi_cipher.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, HiSilicon Limited + */ +#ifndef __HISI_CIPHER_H__ +#define __HISI_CIPHER_H__ + +#include +#include + +enum C_ALG { + C_ALG_DES = 0x0, + C_ALG_3DES = 0x1, + C_ALG_AES = 0x2, + C_ALG_SM4 = 0x3, +}; + +enum C_MODE { + C_MODE_ECB = 0x0, + C_MODE_CBC, + C_MODE_CFB, + C_MODE_OFB, + C_MODE_CTR, + C_MODE_CCM, + C_MODE_GCM, + C_MODE_XTS, + C_MODE_CBC_CS = 0x9, +}; + +#define DES_KEY_SIZE 8 +#define SEC_3DES_2KEY_SIZE (2 * DES_KEY_SIZE) +#define SEC_3DES_3KEY_SIZE (3 * DES_KEY_SIZE) +#define SEC_SM4_XTS_KEY_SIZE 32 +#define SEC_SM4_ECB_KEY_SIZE 16 +#define SEC_MAX_CIPHER_KEY_SIZE 64 +#define MAX_CIPHER_LENGTH 16776704 +#define MIN_CIPHER_LENGTH 16 + +#define DES_CBC_IV_SIZE 8 +#define AES_SM4_IV_SIZE 16 +#define SEC_MAX_IV_SIZE 16 +#define CTR_MODE_LEN_SHIFT 4 +#define CTR_128BIT_COUNTER 16 +#define AES_SM4_BLOCK_SIZE 16 +#define LEFT_MOST_BIT 7 + +static inline uint32_t multiple_round(uint32_t x, uint32_t y) +{ + uint32_t res = 0; + + if (ADD_OVERFLOW(x, y - 1, &res)) + res = UINT32_MAX; + + return res / y; +} + +struct sec_cipher_ctx { + uint8_t key[SEC_MAX_CIPHER_KEY_SIZE]; + uint8_t iv[SEC_MAX_IV_SIZE]; + uint64_t key_dma; + uint64_t iv_dma; + uint8_t *in; + uint64_t in_dma; + uint8_t *out; + uint64_t out_dma; + struct hisi_qp *qp; + size_t offs; + uint32_t len; + uint8_t alg; + uint8_t mode; + uint8_t iv_len; + uint8_t key_len; /* cipher key len */ + uint8_t c_key_len; /* cipher key type */ + bool encrypt; +}; + +#endif diff --git a/core/drivers/crypto/hisilicon/include/hisi_qm.h b/core/drivers/crypto/hisilicon/include/hisi_qm.h new file mode 100644 index 00000000000..344903dfcfa --- /dev/null +++ b/core/drivers/crypto/hisilicon/include/hisi_qm.h @@ -0,0 +1,276 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, Huawei Technologies Co., Ltd + */ +#ifndef __HISI_QM_H__ +#define __HISI_QM_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HISI_QM_HW_V2 0x21 +#define HISI_QM_HW_V3 0x30 +#define HISI_QM_MAX_VFS_NUM 63 +#define HISI_QM_PF_Q_BASE 0 +#define HISI_QM_PF_Q_NUM 64 +#define HISI_QM_VF_Q_NUM 15 +#define HISI_QM_Q_DEPTH 8 +#define PHASE_DEFAULT_VAL 0x1 + +#define HISI_QM_ABNML_INT_MASK 0x100004 +#define HISI_QM_ABNML_INT_MASK_CFG 0x7fff +#define HISI_QM_ABNML_INT_SRC 0x100000 +#define HISI_QM_HPRE_NFE_INT_MASK 0x6fb7 +#define HISI_QM_SEC_NFE_INT_MASK 0x6ff7 +#define HISI_QM_INVALID_DB BIT(12) +#define HISI_QM_REVISON_ID_BASE 0x1000dc +#define HISI_QM_REVISON_ID_MASK GENMASK_32(7, 0) +#define POLL_PERIOD 10 +#define POLL_TIMEOUT 1000 +#define HISI_QM_RECV_SYNC_TIMEOUT 0xfffffff +#define HISI_QM_ALIGN128 128 +#define HISI_QM_ALIGN32 32 + +enum qm_fun_type { + HISI_QM_HW_PF, + HISI_QM_HW_VF, +}; + +enum qm_sq_type { + HISI_QM_CHANNEL_TYPE0 = 0, + HISI_QM_CHANNEL_TYPE1, + HISI_QM_CHANNEL_TYPE2, +}; + +struct qm_sqc { + uint16_t head; + uint16_t tail; + uint32_t base_l; + uint32_t base_h; + /* + * qes : 12 + * sqe : 4 + * rsv(stash_nid/stash_en) : 16 + */ + uint32_t dw3; + uint16_t rand_data; + uint16_t rsv0; + uint16_t pasid; + /* + * rsv : 5 + * head_sig : 1 + * tail_sig : 1 + * pasid_en : 1 + * rsv : 8 + */ + uint16_t w11; + uint16_t cq_num; + /* + * priority(Credit): 4 + * order(order/fc/close/rsv) : 4 + * type : 4 + * rsv : 4 + */ + uint16_t w13; + uint32_t rsv1; +}; + +struct qm_cqc { + uint16_t head; + uint16_t tail; + uint32_t base_l; + uint32_t base_h; + /* + * qes : 12 + * cqe_size : 4 + * rsv(stash_nid/stash_en) : 16 + */ + uint32_t dw3; + uint16_t rand_data; + uint16_t rsv0; + uint16_t pasid; + /* + * pasid_en : 1 + * rsv : 4 + * head_sig : 1 + * tail_sig : 1 + * rsv : 9 + */ + uint16_t w11; + /* + * phase : 1 + * c_flag : 1 + * stash_vld : 1 + */ + uint32_t dw6; + uint32_t rsv1; +}; + +struct qm_cqe { + uint32_t rsv0; + uint16_t cmd_id; + uint16_t rsv1; + uint16_t sq_head; + uint16_t sq_id; + uint16_t rsv2; + /* + * p : 1 + * status : 15 + */ + uint16_t w7; +}; + +struct hisi_qp { + struct hisi_qm *qm; + uint32_t qp_id; + uint8_t sq_type; + uint16_t sq_tail; + uint16_t cq_head; + bool cqc_phase; + + void *sqe; + struct qm_cqe *cqe; + paddr_t sqe_dma; + paddr_t cqe_dma; + + enum hisi_drv_status (*fill_sqe)(void *sqe, void *msg); + enum hisi_drv_status (*parse_sqe)(void *sqe, void *msg); +}; + +struct hisi_qm { + enum qm_fun_type fun_type; + vaddr_t io_base; + uint32_t io_size; + uint32_t vfs_num; + uint32_t version; + + struct qm_sqc *sqc; + struct qm_cqc *cqc; + paddr_t sqc_dma; + paddr_t cqc_dma; + uint32_t sqe_size; + uint32_t sqe_log2_size; + uint32_t qp_base; + uint32_t qp_num; + uint32_t qp_in_used; + uint32_t qp_idx; + struct hisi_qp *qp_array; + struct mutex qp_lock; /* protect the qp instance */ + + enum hisi_drv_status (*dev_status_check)(struct hisi_qm *qm); +}; + +enum hisi_drv_status { + HISI_QM_DRVCRYPT_NO_ERR = 0, + HISI_QM_DRVCRYPT_FAIL = 1, + HISI_QM_DRVCRYPT_EIO = 5, + HISI_QM_DRVCRYPT_EAGAIN = 11, + HISI_QM_DRVCRYPT_ENOMEM = 12, + HISI_QM_DRVCRYPT_EFAULT = 14, + HISI_QM_DRVCRYPT_EBUSY = 16, + HISI_QM_DRVCRYPT_ENODEV = 19, + HISI_QM_DRVCRYPT_EINVAL = 22, + HISI_QM_DRVCRYPT_ETMOUT = 110, + HISI_QM_DRVCRYPT_ENOPROC, + HISI_QM_DRVCRYPT_IN_EPARA, + HISI_QM_DRVCRYPT_VERIFY_ERR, + HISI_QM_DRVCRYPT_HW_EACCESS, +}; + +#define readl_relaxed_poll_timeout(addr, val, cond, delay_us, timeout_us) \ +({ \ + uint64_t timeout = 0; \ + uint64_t _delay_us = delay_us; \ + bool flag = false; \ + while (timeout < (timeout_us)) { \ + (val) = io_read32(addr); \ + if (cond) { \ + flag = true; \ + break; \ + } \ + timeout += (_delay_us); \ + udelay(_delay_us); \ + } \ + (flag) ? HISI_QM_DRVCRYPT_NO_ERR : HISI_QM_DRVCRYPT_ETMOUT; \ +}) + +struct acc_device { + struct hisi_qm qm; + vaddr_t io_base; + uint32_t io_size; + uint32_t vfs_num; + uint32_t endian; + enum qm_fun_type fun_type; + SLIST_ENTRY(acc_device) link; +}; + +/** + *@Description: Get the version information of QM hardware + *@param qm: Handle of Queue Management module + */ +void hisi_qm_get_version(struct hisi_qm *qm); + +/** + *@Description: Init QM for Kunpeng drv + *@param qm: Handle of Queue Management module + *@return success: 0，fail: HISI_QM_DRVCRYPT_EBUSY/HISI_QM_DRVCRYPT_EINVAL + */ +enum hisi_drv_status hisi_qm_init(struct hisi_qm *qm); + +/** + *@Description:deinit QM for Kunpeng drv + *@param qm: Handle of Queue Management module + */ +void hisi_qm_uninit(struct hisi_qm *qm); + +/** + *@Description: Start QM for Kunpeng drv + *@param qm: Handle of Queue Management module + */ +enum hisi_drv_status hisi_qm_start(struct hisi_qm *qm); + +/** + *@Description: Config QM for Kunpeng drv + *@param qm: Handle of Queue Management module + */ +void hisi_qm_dev_init(struct hisi_qm *qm); + +/** + *@Description: Create Queue Pair, allocated to PF/VF for configure + * and service use. Each QP includes one SQ and one CQ + *@param qm: Handle of Queue Management module + *@param sq_type: Accelerator specific algorithm type in sqc + *@return success: Handle of QP，fail: NULL + */ +struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, uint8_t sq_type); + +/** + *@Description:Release Queue Pair + *@param qp: Handle of Queue Pair + */ +void hisi_qm_release_qp(struct hisi_qp *qp); + +/** + *@Description: Send SQE(Submmision Queue Element) to Kunpeng dev + *@param qp: Handle of Queue Pair + *@param msg: The message + *@return success: 0，fail: HISI_QM_DRVCRYPT_EINVAL + */ +enum hisi_drv_status hisi_qp_send(struct hisi_qp *qp, void *msg); + +/** + *@Description: Recevice result from Kunpeng dev + *@param qp: Handle of Queue Pair + *@param msg: The message + *@return success: 0，fail: HISI_QM_DRVCRYPT_EINVAL/ETMOUT + */ +enum hisi_drv_status hisi_qp_recv_sync(struct hisi_qp *qp, void *msg); + +#endif diff --git a/core/drivers/crypto/hisilicon/include/hisi_sec.h b/core/drivers/crypto/hisilicon/include/hisi_sec.h new file mode 100644 index 00000000000..0e57540d533 --- /dev/null +++ b/core/drivers/crypto/hisilicon/include/hisi_sec.h @@ -0,0 +1,347 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (c) 2022, HiSilicon Limited + */ +#ifndef __HISI_SEC_H__ +#define __HISI_SEC_H__ + +#include "hisi_qm.h" + +/* + * Version number maintenance rule: + * first.second.third: year.month.change num + */ +#define SEC_MODULE_VERSION "22.11.2" + +#ifdef CFG_HISILICON_ACC_V3 +#define SEC_BAR 0x160000000 +#else +#define SEC_BAR 0x141800000 +#endif +#define SEC_SIZE 0x400000 + +#define SEC_SQE_SIZE 128 +#define SEC_SQE_LOG2_SIZE 7 + +#define SEC_PF_ABNORMAL_INT_SOURCE_REG 0x0010 +#define SEC_NFE_ERROR_MASK 0x24 +#define SEC_SQE_ICV_SHIFT 1 +#define SEC_SQE_FLAG_SHIFT 7 +#define SEC_MAC_TO_DDR 0x1 +#define SEC_PBKDF2 0x8 +#define MAC_LEN 0x8 +#define MAC_LEN_HMAC_SHA224 0x7 +#define AUTH_KEY_LEN 0x8 +#define C_NUM 10000 +#define PASS_WORD_LEN 0x20 +#define GET_IMG_ROTKEY_AP 0x6 +#define SEC_PBKDF2_SUCC 0x81 +#define SEC_HW_TASK_DONE 0x1 +#define SQE_BYTES_NUMS 128 +#define SEC_USE_HUK 0x1 +#define AES_KEYSIZE_128 16 +#define AES_KEYSIZE_192 24 +#define AES_KEYSIZE_256 32 +#define XTS_KEYSIZE_256 64 +#define XTS_KEYSIZE_128 32 +#define XTS_CKEY_LEN_128_BIT 0 +#define XTS_CKEY_LEN_256_BIT 2 + +#define TYPE_ENCRYPTIN 0 +#define SEC_CIPHER_THEN_DIGEST 0 +#define SEC_DIGEST_THEN_CIPHER 1 + +#define ECB_CBC_SRC_ALIGN_MASK 0xf +#define CTR_SRC_ALIGN_MASK 0xf +#define CTR_SRC_BLOCK_SIZE 0x10 +#define BYTE_BITS 0x8 + +struct hisi_sec_sqe_type2 { + uint32_t nonce_len : 4; + uint32_t huk : 1; + uint32_t key_s : 1; + uint32_t ci_gen : 2; + uint32_t ai_gen : 2; + uint32_t a_pad : 2; + uint32_t c_s : 2; + uint32_t rsvd1 : 2; + uint32_t rhf : 1; + uint32_t c_key_type : 2; + uint32_t a_key_type : 2; + uint32_t write_frame_len : 3; + uint32_t cal_iv_addr_en : 1; + uint32_t tls_up : 1; + uint32_t rsvd0 : 5; + uint32_t inveld : 1; + uint32_t mac_len : 5; + uint32_t a_key_len : 6; + uint32_t a_alg : 6; + uint32_t rsvd3 : 15; + uint32_t c_icv_len : 6; + uint32_t c_width : 3; + uint32_t c_key_len : 3; + uint32_t c_mode : 4; + uint32_t c_alg : 4; + uint32_t rsvd4 : 12; + uint32_t a_len : 24; + uint32_t iv_offset_l : 8; + uint32_t c_len : 24; + uint32_t iv_offset_h : 8; + uint32_t auth_src_offset : 16; + uint32_t cipher_src_offset : 16; + uint32_t cs_ip_header_offset : 16; + uint32_t cs_udp_header_offset : 16; + uint32_t pass_word_len : 16; + uint32_t dk_len : 16; + uint32_t salt3 : 8; + uint32_t salt2 : 8; + uint32_t salt1 : 8; + uint32_t salt0 : 8; + uint32_t tag : 16; + uint32_t rsvd5 : 16; + uint32_t c_pad_type : 4; + uint32_t c_pad_len : 8; + uint32_t c_pad_data_type : 4; + uint32_t c_pad_len_field : 2; + uint32_t rsvd6 : 14; + uint32_t long_a_data_len_l; + uint32_t long_a_data_len_h; + uint32_t a_ivin_addr_l; + uint32_t a_ivin_addr_h; + uint32_t a_key_addr_l; + uint32_t a_key_addr_h; + uint32_t mac_addr_l; + uint32_t mac_addr_h; + uint32_t c_ivin_addr_l; + uint32_t c_ivin_addr_h; + uint32_t c_key_addr_l; + uint32_t c_key_addr_h; + uint32_t data_src_addr_l; + uint32_t data_src_addr_h; + uint32_t data_dst_addr_l; + uint32_t data_dst_addr_h; + uint32_t done : 1; + uint32_t icv : 3; + uint32_t rsvd11 : 3; + uint32_t flag : 4; + uint32_t rsvd10 : 5; + uint32_t error_type : 8; + uint32_t warning_type : 8; + uint32_t mac_i3 : 8; + uint32_t mac_i2 : 8; + uint32_t mac_i1 : 8; + uint32_t mac_i0 : 8; + uint32_t check_sum_i : 16; + uint32_t tls_pad_len_i : 8; + uint32_t rsvd12 : 8; + uint32_t counter; +}; + +struct hisi_sec_sqe { + uint32_t type : 4; + uint32_t cipher : 2; + uint32_t auth : 2; + uint32_t seq : 1; + uint32_t de : 2; + uint32_t scene : 4; + uint32_t src_addr_type : 3; + uint32_t dst_addr_type : 3; + uint32_t mac_addr_type : 3; + uint32_t rsvd0 : 8; + struct hisi_sec_sqe_type2 type2; /* the other sense */ +}; + +struct bd3_auth_key_iv { + uint32_t a_key_addr_l; + uint32_t a_key_addr_h; + uint32_t a_ivin_addr_l; + uint32_t a_ivin_addr_h; + uint32_t rsvd0; + uint32_t rsvd1; +}; + +struct bd3_ipsec_scene { + uint32_t c_ivin_addr_l; + uint32_t c_ivin_addr_h; + uint32_t c_s : 2; + uint32_t deal_esp_ah : 4; + uint32_t protocol_type : 4; + uint32_t mode : 2; + uint32_t ip_type : 2; + uint32_t mac_sel : 1; + uint32_t rsvd0 : 1; + uint32_t next_header : 8; + uint32_t pad_len : 8; + uint32_t iv_offset : 16; + uint32_t rsvd1 : 16; + uint32_t cs_ip_header_offset : 16; + uint32_t cs_udp_header_offset : 16; +}; + +struct bd3_pbkdf2_scene { + uint32_t c_ivin_addr_l; + uint32_t c_ivin_addr_h; + uint32_t pbkdf2_salt_len : 24; + uint32_t rsvd0 : 8; + uint32_t c_num : 24; + uint32_t rsvd1 : 8; + uint32_t pass_word_len : 16; + uint32_t dk_len : 16; +}; + +struct bd3_stream_scene { + uint32_t c_ivin_addr_l; + uint32_t c_ivin_addr_h; + uint32_t long_a_data_len_l; + uint32_t long_a_data_len_h; + uint32_t auth_pad : 2; + uint32_t stream_protocol : 3; + uint32_t mac_sel : 1; + uint32_t rsvd0 : 2; + uint32_t plaintext_type : 8; + uint32_t pad_len_1p3 : 16; +}; + +struct bd3_check_sum { + uint32_t check_sum_i : 16; + uint32_t tls_pad_len_i : 8; + uint32_t rsvd0 : 8; +}; + +struct hisi_sec_bd3_sqe { + uint32_t type : 4; + uint32_t inveld : 1; + uint32_t scene : 4; + uint32_t de : 2; + uint32_t src_addr_type : 3; + uint32_t dst_addr_type : 3; + uint32_t mac_addr_type : 3; + uint32_t rsvd : 12; + + uint32_t cipher : 2; + uint32_t ci_gen : 2; + uint32_t c_icv_len : 6; + uint32_t c_width : 3; + uint32_t c_key_len : 3; + uint32_t c_mode : 4; + uint32_t c_alg : 4; + uint32_t nonce_len : 4; + uint32_t rsv : 1; + uint32_t cal_iv_addr_en : 1; + uint32_t seq : 1; + uint32_t rsvd0 : 1; + + uint32_t tag_l; + uint32_t tag_h; + uint32_t data_src_addr_l; + uint32_t data_src_addr_h; + + struct bd3_auth_key_iv auth_key_iv; + + uint32_t c_key_addr_l; + uint32_t c_key_addr_h; + uint32_t auth : 2; + uint32_t ai_gen : 2; + uint32_t mac_len : 5; + uint32_t a_key_len : 6; + uint32_t a_alg : 6; + uint32_t key_sel : 4; + uint32_t ctr_counter_mode : 2; + uint32_t sva_prefetch : 1; + uint32_t key_wrap_num : 3; + uint32_t update_key : 1; + + uint32_t salt3 : 8; + uint32_t salt2 : 8; + uint32_t salt1 : 8; + uint32_t salt0 : 8; + uint32_t auth_src_offset : 16; + uint32_t cipher_src_offset : 16; + uint32_t a_len : 24; + uint32_t auth_key_offset : 8; + uint32_t c_len : 24; + uint32_t auth_ivin_offset : 8; + uint32_t data_dst_addr_l; + uint32_t data_dst_addr_h; + uint32_t mac_addr_l; + uint32_t mac_addr_h; + + union { + struct bd3_ipsec_scene ipsec_scene; + struct bd3_pbkdf2_scene pbkdf2_scene; + struct bd3_stream_scene stream_scene; + }; + + uint32_t done : 1; + uint32_t icv : 3; + uint32_t csc : 3; + uint32_t flag : 4; + uint32_t dc : 3; + uint32_t rsvd10 : 2; + uint32_t error_type : 8; + uint32_t warning_type : 8; + union { + uint32_t mac_i; + uint32_t kek_key_addr_l; + }; + union { + uint32_t kek_key_addr_h; + struct bd3_check_sum check_sum; + }; + uint32_t counter; +}; + +enum { + NO_CIPHER, + CIPHER_ENCRYPT, + CIPHER_DECRYPT, + REPORT_COPY, +}; + +enum sec_bd_type { + BD_TYPE1 = 0x1, + BD_TYPE2 = 0x2, + BD_TYPE3 = 0x3, +}; + +enum CKEY_LEN { + CKEY_LEN_128_BIT = 0x0, + CKEY_LEN_192_BIT = 0x1, + CKEY_LEN_256_BIT = 0x2, + CKEY_LEN_SM4 = 0x0, + CKEY_LEN_DES = 0x1, + CKEY_LEN_3DES_3KEY = 0x1, + CKEY_LEN_3DES_2KEY = 0x3, +}; + +enum { + SCENE_NOTHING = 0x0, + SCENE_IPSEC = 0x1, + SCENE_SSL_TLS = 0x3, + SCENE_DTLS = 0x4, + SCENE_STORAGE = 0x5, + SCENE_NAS = 0x6, + SCENE_STREAM = 0x7, + SCENE_PBKDF2 = 0x8, + SCENE_SMB = 0x9, +}; + +enum { + DATA_DST_ADDR_DISABLE, + DATA_DST_ADDR_ENABLE, +}; + +enum hisi_buff_type { + HISI_FLAT_BUF, + HISI_SGL_BUF, +}; + +/** + *@Description: Create Queue Pair for SEC, allocated to PF/VF for configure + * and service use. Each QP includes one SQ and one CQ + *@param sq_type: Accelerator specific algorithm type in sqc + *@return success: Handle of QP，fail: NULL + */ +struct hisi_qp *hisi_sec_create_qp(uint8_t sq_type); + +#endif diff --git a/core/drivers/crypto/hisilicon/sec/hisi_sec.c b/core/drivers/crypto/hisilicon/sec/hisi_sec.c new file mode 100644 index 00000000000..4be69909855 --- /dev/null +++ b/core/drivers/crypto/hisilicon/sec/hisi_sec.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright 2022 HiSilicon Limited. + * Kunpeng hardware accelerator sec module init. + */ +#include +#include "hisi_sec.h" + +#define AM_CFG_SINGLE_PORT_MAX_TRANS 0x300014 +#define SEC_CORE_INT_MASK 0x301000 +#define SEC_CORE_INT_SOURCE 0x301010 +#define SEC_RAS_CE_ENABLE 0x301050 +#define SEC_RAS_FE_ENABLE 0x301054 +#define SEC_RAS_NFE_ENABLE 0x301058 +#define SEC_MEM_START_INIT 0x301100 +#define SEC_MEM_INIT_DONE 0x301104 +#define SEC_CONTROL_REG 0x301200 +#define SEC_INTERFACE_USER_CTRL0 0x301220 +#define SEC_INTERFACE_USER_CTRL1 0x301224 +#define SEC_SAA_EN 0x301270 +#define SEC_BD_ERR_CHK_EN0 0x301380 +#define SEC_BD_ERR_CHK_EN1 0x301384 +#define SEC_BD_ERR_CHK_EN3 0x30138c +#define SEC_DYNAMIC_GATE_V3 0x30121c +#define SEC_CORE_AUTO_GATE_V3 0x30212c +#define SEC_INTERFACE_USER_CTRL0_V3 0x302220 +#define SEC_INTERFACE_USER_CTRL1_V3 0x302224 +#define SEC_SINGLE_PORT_MAX_TRANS 0x2060 +#define SEC_ABNML_INT_DISABLE 0x0 +#define SEC_RAS_CE_ENB_MASK 0x88 +#define SEC_RAS_FE_ENB_MASK 0x0 +#define SEC_RAS_NFE_ENB_MASK 0x177 +#define SEC_CLK_GATE_ENABLE BIT(3) +#define SEC_DYNAMIC_GATE_EN 0x7bff +#define SEC_CORE_AUTO_GATE_EN GENMASK_32(3, 0) +#define SEC_TRNG_EN_MASK BIT(8) +#define SEC_SAA_ENABLE 0x17f +#define SEC_SAA_ENABLE_V3 0xf +#define SEC_BD_ERR_CHK0 0xefffffff +#define SEC_BD_ERR_CHK1 0x7ffff7fd +#define SEC_BD_ERR_CHK3 0xffffbfff +#define SEC_USER0_CFG 0x20200 +#define SEC_USER0_SMMU_NORMAL (BIT(23) | BIT(15)) +#define SEC_USER1_CFG 0x12141214 +#define SEC_USER1_SMMU_NORMAL (BIT(31) | BIT(23) | BIT(15) | BIT(7)) +#define SEC_USER0_CFG_V3 0x20200 +#define SEC_USER1_CFG_V3 0x8c494 +#define SEC_LITTLE_ENDIAN 0 + +static SLIST_HEAD(, acc_device) sec_list = SLIST_HEAD_INITIALIZER(sec_list); + +struct hisi_qp *hisi_sec_create_qp(uint8_t sq_type) +{ + struct acc_device *sec_dev = NULL; + struct acc_device *cur_dev = NULL; + struct hisi_qm *qm = NULL; + uint32_t max_qp_num = 0; + uint32_t free_qp_num = 0; + + SLIST_FOREACH(cur_dev, &sec_list, link) { + qm = &cur_dev->qm; + free_qp_num = (qm->fun_type == HISI_QM_HW_PF ? + HISI_QM_PF_Q_NUM : HISI_QM_VF_Q_NUM) - qm->qp_in_used; + if (free_qp_num > max_qp_num) { + max_qp_num = free_qp_num; + sec_dev = cur_dev; + } + } + + if (!sec_dev) { + EMSG("No available sec device"); + return NULL; + } + + return hisi_qm_create_qp(&sec_dev->qm, sq_type); +} + +static void sec_disable_clock_gate(struct hisi_qm *qm) +{ + /* HISI_QM_HW_V2 version need to close clock gating */ + io_clrbits32(qm->io_base + SEC_CONTROL_REG, SEC_CLK_GATE_ENABLE); +} + +static void sec_enable_clock_gate(struct hisi_qm *qm) +{ + if (qm->version == HISI_QM_HW_V2) + return; + + io_setbits32(qm->io_base + SEC_CONTROL_REG, SEC_CLK_GATE_ENABLE); + io_write32(qm->io_base + SEC_DYNAMIC_GATE_V3, SEC_DYNAMIC_GATE_EN); + io_write32(qm->io_base + SEC_CORE_AUTO_GATE_V3, SEC_CORE_AUTO_GATE_EN); +} + +static enum hisi_drv_status sec_engine_init(struct acc_device *sec_dev) +{ + enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; + struct hisi_qm *qm = &sec_dev->qm; + uint32_t val = 0; + + if (qm->fun_type == HISI_QM_HW_VF) + return HISI_QM_DRVCRYPT_NO_ERR; + + sec_disable_clock_gate(qm); + hisi_qm_dev_init(qm); + + io_write32(qm->io_base + SEC_MEM_START_INIT, 0x1); + ret = readl_relaxed_poll_timeout(qm->io_base + SEC_MEM_INIT_DONE, val, + val & 0x1, POLL_PERIOD, POLL_TIMEOUT); + if (ret) { + EMSG("Fail to init sec mem"); + return ret; + } + + io_setbits32(qm->io_base + SEC_CONTROL_REG, SEC_TRNG_EN_MASK); + + if (qm->version == HISI_QM_HW_V2) { + /* smmu bypass */ + io_write32(qm->io_base + SEC_INTERFACE_USER_CTRL0, + SEC_USER0_CFG); + io_write32(qm->io_base + SEC_INTERFACE_USER_CTRL1, + SEC_USER1_CFG); + io_write32(qm->io_base + AM_CFG_SINGLE_PORT_MAX_TRANS, + SEC_SINGLE_PORT_MAX_TRANS); + io_write32(qm->io_base + SEC_SAA_EN, SEC_SAA_ENABLE); + /* HW V2 enable sm4 extra mode, as ctr/ecb */ + io_write32(qm->io_base + SEC_BD_ERR_CHK_EN0, SEC_BD_ERR_CHK0); + /* Enable sm4 xts mode multiple iv */ + io_write32(qm->io_base + SEC_BD_ERR_CHK_EN1, SEC_BD_ERR_CHK1); + io_write32(qm->io_base + SEC_BD_ERR_CHK_EN3, SEC_BD_ERR_CHK3); + } else { + /* cmd_type is controlled by hac subctrl, default normal */ + io_write32(qm->io_base + SEC_INTERFACE_USER_CTRL0_V3, + SEC_USER0_CFG_V3); + io_write32(qm->io_base + SEC_INTERFACE_USER_CTRL1_V3, + SEC_USER1_CFG_V3); + io_write32(qm->io_base + SEC_SAA_EN, SEC_SAA_ENABLE_V3); + } + + io_write32(qm->io_base + SEC_RAS_CE_ENABLE, SEC_RAS_CE_ENB_MASK); + io_write32(qm->io_base + SEC_RAS_FE_ENABLE, SEC_RAS_FE_ENB_MASK); + io_write32(qm->io_base + SEC_RAS_NFE_ENABLE, SEC_RAS_NFE_ENB_MASK); + io_write32(qm->io_base + SEC_CORE_INT_MASK, SEC_ABNML_INT_DISABLE); + + io_setbits32(qm->io_base + SEC_CONTROL_REG, sec_dev->endian); + + sec_enable_clock_gate(qm); + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static enum hisi_drv_status sec_dev_status_check(struct hisi_qm *qm) +{ + uint32_t val = 0; + + val = io_read32(qm->io_base + SEC_CORE_INT_SOURCE); + if (val & SEC_RAS_NFE_ENB_MASK) { + EMSG("SEC NFE RAS happened, need to reset"); + return HISI_QM_DRVCRYPT_HW_EACCESS; + } + + val = io_read32(qm->io_base + HISI_QM_ABNML_INT_SRC); + if (val) { + if (val & HISI_QM_SEC_NFE_INT_MASK) + EMSG("QM NFE RAS happened, need to reset"); + + if (val & HISI_QM_INVALID_DB) { + EMSG("QM invalid db happened, please check"); + io_write32(qm->io_base + HISI_QM_ABNML_INT_SRC, + HISI_QM_INVALID_DB); + } + + return HISI_QM_DRVCRYPT_HW_EACCESS; + } + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static enum hisi_drv_status sec_qm_init(struct acc_device *sec_dev) +{ + struct hisi_qm *qm = &sec_dev->qm; + + if (cpu_mmu_enabled()) { + qm->io_base = (vaddr_t)phys_to_virt_io(sec_dev->io_base, + sec_dev->io_size); + if (!qm->io_base) { + EMSG("Fail to get qm io_base"); + return HISI_QM_DRVCRYPT_EFAULT; + } + } else { + qm->io_base = sec_dev->io_base; + } + + qm->fun_type = sec_dev->fun_type; + qm->vfs_num = sec_dev->vfs_num; + qm->sqe_size = SEC_SQE_SIZE; + qm->sqe_log2_size = SEC_SQE_LOG2_SIZE; + if (qm->fun_type == HISI_QM_HW_PF) { + hisi_qm_get_version(qm); + IMSG("SEC hardware version is 0x%" PRIx32, qm->version); + qm->qp_base = HISI_QM_PF_Q_BASE; + qm->qp_num = HISI_QM_PF_Q_NUM; + qm->dev_status_check = sec_dev_status_check; + } + + return hisi_qm_init(qm); +} + +static struct acc_device *sec_pre_init(void) +{ + struct acc_device *sec_dev = NULL; + + sec_dev = (struct acc_device *)calloc(1, sizeof(*sec_dev)); + if (!sec_dev) { + EMSG("Fail to alloc sec_dev"); + return NULL; + } + + sec_dev->io_base = SEC_BAR; + sec_dev->io_size = SEC_SIZE; + sec_dev->vfs_num = 0; + sec_dev->endian = SEC_LITTLE_ENDIAN; + sec_dev->fun_type = HISI_QM_HW_PF; + SLIST_INSERT_HEAD(&sec_list, sec_dev, link); + + return sec_dev; +} + +static TEE_Result sec_probe(void) +{ + struct acc_device *sec_dev = NULL; + struct hisi_qm *qm = NULL; + TEE_Result ret = TEE_SUCCESS; + + IMSG("SEC driver init start, version %s", SEC_MODULE_VERSION); + sec_dev = sec_pre_init(); + if (!sec_dev) + return TEE_ERROR_STORAGE_NO_SPACE; + + qm = &sec_dev->qm; + ret = sec_qm_init(sec_dev); + if (ret) { + EMSG("Fail to init sec qm, ret=%d\n", ret); + goto err_with_pre_init; + } + + ret = sec_engine_init(sec_dev); + if (ret) { + EMSG("fail to init engine, ret=%d\n", ret); + goto err_with_qm_init; + } + + ret = hisi_qm_start(qm); + if (ret) { + EMSG("Fail to start qm, ret=%d\n", ret); + goto err_with_qm_init; + } + + IMSG("SEC driver init done"); + return TEE_SUCCESS; + +err_with_qm_init: + hisi_qm_uninit(qm); +err_with_pre_init: + SLIST_REMOVE_HEAD(&sec_list, link); + free(sec_dev); + + return TEE_ERROR_GENERIC; +} + +driver_init(sec_probe); diff --git a/core/drivers/crypto/hisilicon/sec/sec_cipher.c b/core/drivers/crypto/hisilicon/sec/sec_cipher.c new file mode 100644 index 00000000000..ee0b1a7585c --- /dev/null +++ b/core/drivers/crypto/hisilicon/sec/sec_cipher.c @@ -0,0 +1,774 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2022, HiSilicon Limited + */ +#include +#include "hisi_cipher.h" +#include "hisi_sec.h" + +static TEE_Result sec_do_cipher_task(struct hisi_qp *qp, void *msg) +{ + TEE_Result ret = TEE_SUCCESS; + + ret = hisi_qp_send(qp, msg); + if (ret) { + EMSG("Fail to send task, ret=%"PRIu32, ret); + return TEE_ERROR_BAD_STATE; + } + + ret = hisi_qp_recv_sync(qp, msg); + if (ret) { + EMSG("Recv task error, ret=%"PRIu32, ret); + return TEE_ERROR_BAD_STATE; + } + + return TEE_SUCCESS; +} + +/* increment counter (128-bit int) by c */ +static void ctr_iv_inc(uint8_t *counter, uint32_t shift_len) +{ + uint32_t n = CTR_128BIT_COUNTER; + uint8_t *counter1 = counter; + uint32_t c = shift_len; + + do { + --n; + c += counter1[n]; + counter1[n] = (uint8_t)c; + c >>= BYTE_BITS; + } while (n); +} + +static void xts_multi_galois(unsigned char *data) +{ + uint8_t tt = 0; + uint8_t t = 0; + int i = 0; + + for (i = 0; i < AES_SM4_IV_SIZE; i++) { + tt = data[i] >> LEFT_MOST_BIT; + data[i] = ((data[i] << 1) | t) & 0xFF; + t = tt; + } + if (tt) + data[0] ^= 0x87; +} + +static TEE_Result sec_cipher_set_key(struct sec_cipher_ctx *c_ctx, + const uint8_t *key1, const int key1_len, + const uint8_t *key2, const int key2_len); + +/* + * When the IV is delivered by segment, the AES/SM4-ECB is used + * to update the IV to be used next time. + */ +static TEE_Result xts_iv_update(struct sec_cipher_ctx *c_ctx) +{ + size_t xts_key_len = c_ctx->key_len / 2; + struct sec_cipher_ctx ecb_ctx = { }; + TEE_Result ret = TEE_SUCCESS; + size_t i = 0; + + ecb_ctx.alg = c_ctx->alg; + ecb_ctx.mode = C_MODE_ECB; + ret = sec_cipher_set_key(&ecb_ctx, c_ctx->key + xts_key_len, + xts_key_len, NULL, 0); + if (ret) + return ret; + + ecb_ctx.encrypt = true; + ecb_ctx.in = c_ctx->iv; + ecb_ctx.out = c_ctx->iv; + ecb_ctx.in_dma = c_ctx->iv_dma; + ecb_ctx.out_dma = c_ctx->iv_dma; + ecb_ctx.len = c_ctx->iv_len; + + ret = sec_do_cipher_task(c_ctx->qp, &ecb_ctx); + if (ret) { + EMSG("XTS iv enc failed . ret = 0x%" PRIx32, ret); + return ret; + } + + for (i = 0; i < multiple_round(c_ctx->len, AES_SM4_BLOCK_SIZE); i++) + xts_multi_galois(c_ctx->iv); + + ecb_ctx.encrypt = false; + ret = sec_do_cipher_task(c_ctx->qp, &ecb_ctx); + if (ret) + EMSG("XTS iv denc failed . ret = 0x%" PRIx32, ret); + + return ret; +} + +static TEE_Result sec_update_iv(struct sec_cipher_ctx *c_ctx) +{ + TEE_Result ret = TEE_SUCCESS; + size_t offset = 0; + + switch (c_ctx->mode) { + case C_MODE_CBC: + offset = c_ctx->len - c_ctx->iv_len; + if (c_ctx->encrypt && c_ctx->len >= c_ctx->iv_len) + memcpy(c_ctx->iv, c_ctx->out + offset, c_ctx->iv_len); + if (!c_ctx->encrypt && c_ctx->len >= c_ctx->iv_len) + memcpy(c_ctx->iv, c_ctx->in + offset, c_ctx->iv_len); + break; + case C_MODE_CTR: + ctr_iv_inc(c_ctx->iv, c_ctx->len >> CTR_MODE_LEN_SHIFT); + break; + case C_MODE_XTS: + ret = xts_iv_update(c_ctx); + break; + default: + break; + } + + return ret; +} + +static TEE_Result sec_cipher_des_get_c_key_len(const int key_len, + uint8_t *c_key_len) +{ + if (key_len == DES_KEY_SIZE) { + *c_key_len = CKEY_LEN_DES; + } else { + EMSG("Invalid DES key size"); + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static TEE_Result sec_cipher_3des_get_c_key_len(const int key_len, + uint8_t *c_key_len) +{ + if (key_len == SEC_3DES_2KEY_SIZE) { + *c_key_len = CKEY_LEN_3DES_2KEY; + } else if (key_len == SEC_3DES_3KEY_SIZE) { + *c_key_len = CKEY_LEN_3DES_3KEY; + } else { + EMSG("Invalid 3DES key size"); + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static TEE_Result sec_cipher_aes_get_c_key_len(const int key_len, + const uint8_t mode, + uint8_t *c_key_len) +{ + switch (mode) { + case C_MODE_ECB: + case C_MODE_CBC: + case C_MODE_CTR: + case C_MODE_GCM: + switch (key_len) { + case AES_KEYSIZE_128: + *c_key_len = CKEY_LEN_128_BIT; + break; + case AES_KEYSIZE_192: + *c_key_len = CKEY_LEN_192_BIT; + break; + case AES_KEYSIZE_256: + *c_key_len = CKEY_LEN_256_BIT; + break; + default: + EMSG("Invalid AES key size"); + return TEE_ERROR_BAD_PARAMETERS; + } + break; + case C_MODE_XTS: + switch (key_len) { + case XTS_KEYSIZE_128: + *c_key_len = CKEY_LEN_128_BIT; + break; + case XTS_KEYSIZE_256: + *c_key_len = CKEY_LEN_256_BIT; + break; + default: + EMSG("Invalid AES-XTS key size"); + return TEE_ERROR_BAD_PARAMETERS; + } + break; + default: + EMSG("Unsupported AES mode\n"); + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static TEE_Result sec_cipher_sm4_get_c_key_len(const int key_len, + const uint8_t mode, + uint8_t *c_key_len) +{ + switch (mode) { + case C_MODE_ECB: + case C_MODE_CBC: + case C_MODE_CTR: + if (key_len != AES_KEYSIZE_128) { + EMSG("Invalid SM4 key size"); + return TEE_ERROR_BAD_PARAMETERS; + } + *c_key_len = CKEY_LEN_128_BIT; + break; + case C_MODE_XTS: + if (key_len != XTS_KEYSIZE_128) { + EMSG("Invalid SM4-XTS key size"); + return TEE_ERROR_BAD_PARAMETERS; + } + *c_key_len = CKEY_LEN_128_BIT; + break; + default: + EMSG("Unsupported SM4 mode\n"); + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static TEE_Result sec_cipher_set_key(struct sec_cipher_ctx *c_ctx, + const uint8_t *key1, const int key1_len, + const uint8_t *key2, const int key2_len) +{ + int key_len = key1_len + key2_len; + TEE_Result ret = TEE_SUCCESS; + uint8_t c_key_len = 0; + + switch (c_ctx->alg) { + case C_ALG_DES: + ret = sec_cipher_des_get_c_key_len(key_len, &c_key_len); + break; + case C_ALG_3DES: + ret = sec_cipher_3des_get_c_key_len(key_len, &c_key_len); + break; + case C_ALG_AES: + ret = sec_cipher_aes_get_c_key_len(key_len, + c_ctx->mode, &c_key_len); + break; + case C_ALG_SM4: + ret = sec_cipher_sm4_get_c_key_len(key_len, + c_ctx->mode, &c_key_len); + break; + default: + EMSG("Invalid cipher type 0x%" PRIx32, c_ctx->alg); + ret = TEE_ERROR_NOT_IMPLEMENTED; + break; + } + + if (ret) + return ret; + + c_ctx->key_dma = virt_to_phys(c_ctx->key); + c_ctx->key_len = key_len; + c_ctx->c_key_len = c_key_len; + + memcpy(c_ctx->key, key1, key1_len); + memcpy(c_ctx->key + key1_len, key2, key2_len); + + return ret; +} + +static TEE_Result sec_cipher_iv_check(struct sec_cipher_ctx *c_ctx, + const int iv_size) +{ + TEE_Result ret = TEE_SUCCESS; + + switch (c_ctx->mode) { + case C_MODE_ECB: + ret = iv_size == TEE_SUCCESS ? + TEE_SUCCESS : TEE_ERROR_BAD_PARAMETERS; + break; + case C_MODE_CBC: + if (c_ctx->alg == C_ALG_DES || c_ctx->alg == C_ALG_3DES) { + ret = iv_size == DES_CBC_IV_SIZE ? + TEE_SUCCESS : TEE_ERROR_BAD_PARAMETERS; + break; + } + fallthrough; + case C_MODE_XTS: + case C_MODE_CTR: + if (c_ctx->alg == C_ALG_AES || c_ctx->alg == C_ALG_SM4) { + ret = iv_size == AES_SM4_IV_SIZE ? + TEE_SUCCESS : TEE_ERROR_BAD_PARAMETERS; + break; + } + fallthrough; + default: + ret = TEE_ERROR_BAD_PARAMETERS; + break; + } + + if (ret) + EMSG("iv_size check failed"); + + return ret; +} + +static TEE_Result sec_cipher_set_iv(struct sec_cipher_ctx *c_ctx, + const uint8_t *iv, const int iv_len) +{ + TEE_Result ret = TEE_SUCCESS; + + if (!iv && iv_len) { + EMSG("IV is NULL"); + return TEE_ERROR_BAD_PARAMETERS; + } + + ret = sec_cipher_iv_check(c_ctx, iv_len); + if (ret) + return ret; + + c_ctx->iv_len = iv_len; + c_ctx->iv_dma = virt_to_phys(c_ctx->iv); + + memcpy(c_ctx->iv, iv, c_ctx->iv_len); + + return TEE_SUCCESS; +} + +static enum hisi_drv_status sec_cipher_bd_fill(void *bd, void *msg) +{ + struct sec_cipher_ctx *c_ctx = (struct sec_cipher_ctx *)msg; + struct hisi_sec_sqe *sqe = (struct hisi_sec_sqe *)bd; + + sqe->type = BD_TYPE2; + sqe->scene = SCENE_IPSEC; + sqe->de = DATA_DST_ADDR_ENABLE; + sqe->type2.c_len = c_ctx->len; + sqe->src_addr_type = HISI_FLAT_BUF; + sqe->dst_addr_type = HISI_FLAT_BUF; + sqe->type2.c_alg = c_ctx->alg; + sqe->type2.c_mode = c_ctx->mode; + sqe->type2.c_key_len = c_ctx->c_key_len; + + if (c_ctx->encrypt) + sqe->cipher = CIPHER_ENCRYPT; + else + sqe->cipher = CIPHER_DECRYPT; + + reg_pair_from_64(c_ctx->out_dma, &sqe->type2.data_dst_addr_h, + &sqe->type2.data_dst_addr_l); + reg_pair_from_64(c_ctx->in_dma, &sqe->type2.data_src_addr_h, + &sqe->type2.data_src_addr_l); + reg_pair_from_64(c_ctx->key_dma, &sqe->type2.c_key_addr_h, + &sqe->type2.c_key_addr_l); + if (c_ctx->iv_len == 0) + return HISI_QM_DRVCRYPT_NO_ERR; + + reg_pair_from_64(c_ctx->iv_dma, &sqe->type2.c_ivin_addr_h, + &sqe->type2.c_ivin_addr_l); + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static enum hisi_drv_status sec_cipher_bd_parse(void *bd, void *msg __unused) +{ + struct hisi_sec_sqe *sqe = (struct hisi_sec_sqe *)bd; + + if (sqe->type2.done != SEC_HW_TASK_DONE || sqe->type2.error_type) { + EMSG("SEC BD2 fail done=0x%" PRIx32 ", etype=0x%" PRIx32, + sqe->type2.done, sqe->type2.error_type); + return HISI_QM_DRVCRYPT_EINVAL; + } + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static enum hisi_drv_status sec_cipher_bd3_fill(void *bd, void *msg) +{ + struct hisi_sec_bd3_sqe *sqe = (struct hisi_sec_bd3_sqe *)bd; + struct sec_cipher_ctx *c_ctx = (struct sec_cipher_ctx *)msg; + + sqe->type = BD_TYPE3; + sqe->scene = SCENE_IPSEC; + sqe->de = DATA_DST_ADDR_ENABLE; + sqe->c_len = c_ctx->len; + sqe->src_addr_type = HISI_FLAT_BUF; + sqe->dst_addr_type = HISI_FLAT_BUF; + sqe->c_alg = c_ctx->alg; + sqe->c_mode = c_ctx->mode; + sqe->c_key_len = c_ctx->c_key_len; + + if (c_ctx->encrypt) + sqe->cipher = CIPHER_ENCRYPT; + else + sqe->cipher = CIPHER_DECRYPT; + + reg_pair_from_64(c_ctx->out_dma, &sqe->data_dst_addr_h, + &sqe->data_dst_addr_l); + reg_pair_from_64(c_ctx->in_dma, &sqe->data_src_addr_h, + &sqe->data_src_addr_l); + reg_pair_from_64(c_ctx->key_dma, &sqe->c_key_addr_h, + &sqe->c_key_addr_l); + + if (c_ctx->iv_len == 0) + return HISI_QM_DRVCRYPT_NO_ERR; + + reg_pair_from_64(c_ctx->iv_dma, &sqe->ipsec_scene.c_ivin_addr_h, + &sqe->ipsec_scene.c_ivin_addr_l); + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static enum hisi_drv_status sec_cipher_bd3_parse(void *bd, void *msg __unused) +{ + struct hisi_sec_bd3_sqe *sqe = (struct hisi_sec_bd3_sqe *)bd; + + if (sqe->done != SEC_HW_TASK_DONE || sqe->error_type) { + EMSG("SEC BD3 fail done=0x%" PRIx32 ", etype=0x%" PRIx32, + sqe->done, sqe->error_type); + return HISI_QM_DRVCRYPT_EINVAL; + } + + return HISI_QM_DRVCRYPT_NO_ERR; +} + +static TEE_Result cipher_algo_check(uint32_t algo) +{ + switch (algo) { + case TEE_ALG_AES_ECB_NOPAD: + case TEE_ALG_AES_CBC_NOPAD: + case TEE_ALG_AES_CTR: + case TEE_ALG_AES_XTS: + case TEE_ALG_DES_ECB_NOPAD: + case TEE_ALG_DES3_ECB_NOPAD: + case TEE_ALG_DES_CBC_NOPAD: + case TEE_ALG_DES3_CBC_NOPAD: + case TEE_ALG_SM4_CBC_NOPAD: + case TEE_ALG_SM4_ECB_NOPAD: + case TEE_ALG_SM4_CTR: + return TEE_SUCCESS; + default: + return TEE_ERROR_NOT_IMPLEMENTED; + } +} + +static TEE_Result crypto_set_calg(struct sec_cipher_ctx *c_ctx, + const uint32_t alg) +{ + TEE_Result ret = TEE_SUCCESS; + + switch (alg) { + case TEE_MAIN_ALGO_DES: + c_ctx->alg = C_ALG_DES; + break; + case TEE_MAIN_ALGO_DES3: + c_ctx->alg = C_ALG_3DES; + break; + case TEE_MAIN_ALGO_AES: + c_ctx->alg = C_ALG_AES; + break; + case TEE_MAIN_ALGO_SM4: + c_ctx->alg = C_ALG_SM4; + break; + default: + EMSG("Invalid cipher type 0x%"PRIx32, alg); + ret = TEE_ERROR_NOT_IMPLEMENTED; + break; + } + + return ret; +} + +static TEE_Result crypto_set_cmode(struct sec_cipher_ctx *c_ctx, + const uint32_t mode) +{ + TEE_Result ret = TEE_SUCCESS; + + switch (mode) { + case TEE_CHAIN_MODE_ECB_NOPAD: + c_ctx->mode = C_MODE_ECB; + break; + case TEE_CHAIN_MODE_CBC_NOPAD: + c_ctx->mode = C_MODE_CBC; + break; + case TEE_CHAIN_MODE_XTS: + c_ctx->mode = C_MODE_XTS; + break; + case TEE_CHAIN_MODE_CTR: + c_ctx->mode = C_MODE_CTR; + break; + default: + EMSG("Invalid cipher mode type 0x%" PRIx32, mode); + ret = TEE_ERROR_NOT_IMPLEMENTED; + break; + } + + return ret; +} + +static TEE_Result sec_cipher_alloc_ctx(void **ctx, uint32_t algo) +{ + struct sec_cipher_ctx *c_ctx = NULL; + TEE_Result ret = TEE_SUCCESS; + + if (!ctx) { + EMSG("Ctx is NULL"); + return TEE_ERROR_STORAGE_NO_SPACE; + } + + ret = cipher_algo_check(algo); + if (ret) + return ret; + + c_ctx = (struct sec_cipher_ctx *)malloc(sizeof(struct sec_cipher_ctx)); + if (!c_ctx) { + EMSG("c_ctx is NULL"); + return TEE_ERROR_STORAGE_NO_SPACE; + } + + ret = crypto_set_calg(c_ctx, TEE_ALG_GET_MAIN_ALG(algo)); + if (ret) + goto free_c_ctx; + + ret = crypto_set_cmode(c_ctx, TEE_ALG_GET_CHAIN_MODE(algo)); + if (ret) + goto free_c_ctx; + + c_ctx->qp = hisi_sec_create_qp(HISI_QM_CHANNEL_TYPE0); + if (!c_ctx->qp) { + ret = TEE_ERROR_BUSY; + goto free_c_ctx; + } + + if (c_ctx->qp->qm->version == HISI_QM_HW_V2) { + c_ctx->qp->fill_sqe = sec_cipher_bd_fill; + c_ctx->qp->parse_sqe = sec_cipher_bd_parse; + } else { + c_ctx->qp->fill_sqe = sec_cipher_bd3_fill; + c_ctx->qp->parse_sqe = sec_cipher_bd3_parse; + } + + c_ctx->offs = 0; + *ctx = c_ctx; + + return TEE_SUCCESS; + +free_c_ctx: + free(c_ctx); + + return ret; +} + +static void sec_cipher_free_ctx(void *ctx) +{ + struct sec_cipher_ctx *c_ctx = (struct sec_cipher_ctx *)ctx; + + if (!c_ctx) + return; + + hisi_qm_release_qp(c_ctx->qp); + memzero_explicit(c_ctx->key, c_ctx->key_len); + free(c_ctx); +} + +static TEE_Result sec_cipher_init(struct drvcrypt_cipher_init *dinit) +{ + struct sec_cipher_ctx *c_ctx = NULL; + TEE_Result ret = TEE_SUCCESS; + + if (!dinit || !dinit->ctx || !dinit->key1.data) { + EMSG("drvcrypt_cipher init param error"); + return TEE_ERROR_BAD_PARAMETERS; + } + + c_ctx = (struct sec_cipher_ctx *)dinit->ctx; + + ret = sec_cipher_set_key(c_ctx, dinit->key1.data, dinit->key1.length, + dinit->key2.data, dinit->key2.length); + if (ret) + return ret; + + ret = sec_cipher_set_iv(c_ctx, dinit->iv.data, dinit->iv.length); + if (ret) + return ret; + + c_ctx->encrypt = dinit->encrypt; + + return TEE_SUCCESS; +} + +static TEE_Result sec_cipher_cryptlen_check(struct sec_cipher_ctx *c_ctx, + size_t length) +{ + if (c_ctx->mode == C_MODE_XTS && length < AES_SM4_BLOCK_SIZE) { + EMSG("Invalid src len"); + return TEE_ERROR_BAD_PARAMETERS; + } + if ((c_ctx->mode == C_MODE_ECB || c_ctx->mode == C_MODE_CBC) && + (length & (AES_SM4_BLOCK_SIZE - 1))) { + EMSG("Invalid src len"); + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static TEE_Result sec_cipher_param_check(struct drvcrypt_cipher_update *dupdate) +{ + struct sec_cipher_ctx *c_ctx = NULL; + + if (!dupdate || !dupdate->src.data || !dupdate->dst.data || + dupdate->src.length != dupdate->dst.length || + dupdate->src.length > MAX_CIPHER_LENGTH || + !dupdate->src.length) { + EMSG("Dupdate input param error"); + return TEE_ERROR_BAD_PARAMETERS; + } + + c_ctx = (struct sec_cipher_ctx *)dupdate->ctx; + switch (c_ctx->alg) { + case C_ALG_SM4: + case C_ALG_AES: + if (sec_cipher_cryptlen_check(c_ctx, dupdate->src.length)) + return TEE_ERROR_BAD_PARAMETERS; + break; + case C_ALG_DES: + case C_ALG_3DES: + if (dupdate->src.length % TEE_DES_BLOCK_SIZE) { + EMSG("Invalid src len"); + return TEE_ERROR_BAD_PARAMETERS; + } + break; + default: + return TEE_ERROR_BAD_PARAMETERS; + } + + return TEE_SUCCESS; +} + +static TEE_Result sec_alloc_buffer(struct sec_cipher_ctx *c_ctx) +{ + c_ctx->in = (uint8_t *)malloc(c_ctx->len); + if (!c_ctx->in) { + EMSG("Failed to alloc c_in buf"); + return TEE_ERROR_STORAGE_NO_SPACE; + } + + c_ctx->in_dma = virt_to_phys(c_ctx->in); + + c_ctx->out = (uint8_t *)malloc(c_ctx->len); + if (!c_ctx->out) { + EMSG("Failed to alloc c_out buf"); + goto free_c_in; + } + + c_ctx->out_dma = virt_to_phys(c_ctx->out); + + return TEE_SUCCESS; + +free_c_in: + free(c_ctx->in); + return TEE_ERROR_STORAGE_NO_SPACE; +} + +static void sec_free_buffer(struct sec_cipher_ctx *c_ctx) +{ + free(c_ctx->in); + free(c_ctx->out); +} + +static TEE_Result sec_cipher_update(struct drvcrypt_cipher_update *dupdate) +{ + struct sec_cipher_ctx *c_ctx = NULL; + size_t padding_size = 0; + TEE_Result ret = TEE_SUCCESS; + + ret = sec_cipher_param_check(dupdate); + if (ret) + return ret; + + c_ctx = (struct sec_cipher_ctx *)dupdate->ctx; + if (c_ctx->mode == C_MODE_CTR && (c_ctx->offs & CTR_SRC_ALIGN_MASK)) + padding_size = (c_ctx->offs % CTR_SRC_BLOCK_SIZE); + + c_ctx->offs += dupdate->src.length; + c_ctx->len = dupdate->src.length + padding_size; + ret = sec_alloc_buffer(c_ctx); + if (ret) + return ret; + + memzero_explicit(c_ctx->in, padding_size); + memcpy(c_ctx->in + padding_size, + dupdate->src.data, dupdate->src.length); + + ret = sec_do_cipher_task(c_ctx->qp, c_ctx); + if (ret) + goto free_buffer; + + ret = sec_update_iv(c_ctx); + if (ret) { + EMSG("sec_update_iv failed. ret = 0x%" PRIx32, ret); + goto free_buffer; + } + + memcpy(dupdate->dst.data, + c_ctx->out + padding_size, dupdate->src.length); + +free_buffer: + sec_free_buffer(c_ctx); + return ret; +} + +/* + * Finalize of the cipher operation + * + * @ctx Caller context variable or NULL + */ +static void sec_cipher_final(void *ctx __unused) +{ +} + +static void sec_cipher_copy_state(void *dst_ctx, void *src_ctx) +{ + struct sec_cipher_ctx *dst_c_ctx = (struct sec_cipher_ctx *)dst_ctx; + struct sec_cipher_ctx *src_c_ctx = (struct sec_cipher_ctx *)src_ctx; + + dst_c_ctx->alg = src_c_ctx->alg; + dst_c_ctx->mode = src_c_ctx->mode; + dst_c_ctx->encrypt = src_c_ctx->encrypt; + dst_c_ctx->offs = src_c_ctx->offs; + + if (src_c_ctx->key_len) { + dst_c_ctx->key_len = src_c_ctx->key_len; + dst_c_ctx->c_key_len = src_c_ctx->c_key_len; + memcpy(dst_c_ctx->key, src_c_ctx->key, dst_c_ctx->key_len); + dst_c_ctx->key_dma = virt_to_phys(dst_c_ctx->key); + } + + if (src_c_ctx->iv_len) { + dst_c_ctx->iv_len = src_c_ctx->iv_len; + memcpy(dst_c_ctx->iv, src_c_ctx->iv, dst_c_ctx->iv_len); + dst_c_ctx->iv_dma = virt_to_phys(dst_c_ctx->iv); + } +} + +/* + * Registration of the Cipher Driver + */ +static struct drvcrypt_cipher driver_cipher = { + .alloc_ctx = sec_cipher_alloc_ctx, + .free_ctx = sec_cipher_free_ctx, + .init = sec_cipher_init, + .update = sec_cipher_update, + .final = sec_cipher_final, + .copy_state = sec_cipher_copy_state, +}; + +/* + * Initialize the Cipher module + * + */ +static TEE_Result sec_init_cipher(void) +{ + TEE_Result ret = drvcrypt_register_cipher(&driver_cipher); + + if (ret) + EMSG("Sec cipher register failed. ret = 0x%" PRIx32, ret); + + return ret; +} +driver_init(sec_init_cipher); diff --git a/core/drivers/crypto/hisilicon/sec/sub.mk b/core/drivers/crypto/hisilicon/sec/sub.mk new file mode 100644 index 00000000000..9cef92b9837 --- /dev/null +++ b/core/drivers/crypto/hisilicon/sec/sub.mk @@ -0,0 +1,4 @@ +incdirs-y += ../include + +srcs-y += hisi_sec.c +srcs-y += sec_cipher.c diff --git a/core/drivers/crypto/hisilicon/sub.mk b/core/drivers/crypto/hisilicon/sub.mk new file mode 100644 index 00000000000..7cb3f6d4122 --- /dev/null +++ b/core/drivers/crypto/hisilicon/sub.mk @@ -0,0 +1,4 @@ +incdirs-y += include +subdirs-y += sec + +srcs-y += hisi_qm.c diff --git a/core/drivers/crypto/sub.mk b/core/drivers/crypto/sub.mk index 3c26eda79a2..71cb6bd6905 100644 --- a/core/drivers/crypto/sub.mk +++ b/core/drivers/crypto/sub.mk @@ -11,3 +11,5 @@ subdirs-$(CFG_STM32_CRYPTO_DRIVER) += stm32 subdirs-$(CFG_ASPEED_CRYPTO_DRIVER) += aspeed subdirs-$(CFG_VERSAL_CRYPTO_DRIVER) += versal + +subdirs-$(CFG_HISILICON_CRYPTO_DRIVER) += hisilicon