diff --git a/core/arch/arm/include/ffa.h b/core/arch/arm/include/ffa.h index c54479b663e..1a7baccfb1f 100644 --- a/core/arch/arm/include/ffa.h +++ b/core/arch/arm/include/ffa.h @@ -92,6 +92,9 @@ #define FFA_CONSOLE_LOG_32 U(0x8400008A) #define FFA_CONSOLE_LOG_64 U(0xC400008A) +#define FFA_FEATURES_FUNC_ID_MASK BIT32(31) +#define FFA_FEATURES_FEATURE_ID_MASK GENMASK_32(7, 0) + #define FFA_FEATURE_NOTIF_PEND_INTR U(0x1) #define FFA_FEATURE_SCHEDULE_RECV_INTR U(0x2) #define FFA_FEATURE_MANAGED_EXIT_INTR U(0x3) diff --git a/core/arch/arm/include/kernel/thread_spmc.h b/core/arch/arm/include/kernel/thread_spmc.h index ffef0903633..672bc2cc82e 100644 --- a/core/arch/arm/include/kernel/thread_spmc.h +++ b/core/arch/arm/include/kernel/thread_spmc.h @@ -1,11 +1,14 @@ /* SPDX-License-Identifier: BSD-2-Clause */ /* * Copyright (c) 2021, Arm Limited. + * Copyright (c) 2023, Linaro Limited */ #ifndef __KERNEL_THREAD_SPMC_H #define __KERNEL_THREAD_SPMC_H +#include #include +#include #include /* FF-A endpoint base ID when OP-TEE is used as a S-EL1 endpoint */ @@ -37,7 +40,14 @@ TEE_Result spmc_fill_partition_entry(uint32_t ffa_vers, void *buf, size_t blen, int spmc_read_mem_transaction(uint32_t ffa_vers, void *buf, size_t blen, struct ffa_mem_transaction_x *trans); -#if !defined(CFG_CORE_SEL1_SPMC) +#if defined(CFG_CORE_SEL1_SPMC) +void thread_spmc_set_async_notif_intid(int intid); +#else +static inline void __noreturn +thread_spmc_set_async_notif_intid(int intid __unused) +{ + panic(); +} struct mobj_ffa *thread_spmc_populate_mobj_from_rx(uint64_t cookie); void thread_spmc_relinquish(uint64_t memory_region_handle); #endif diff --git a/core/arch/arm/include/optee_ffa.h b/core/arch/arm/include/optee_ffa.h index 8f1603946bf..8db02650be5 100644 --- a/core/arch/arm/include/optee_ffa.h +++ b/core/arch/arm/include/optee_ffa.h @@ -74,7 +74,7 @@ * * Call register usage: * w3: Service ID, OPTEE_FFA_EXCHANGE_CAPABILITIES - * w4-w7: Note used (MBZ) + * w4-w7: Not used (MBZ) * * Return register usage: * w3: Error code, 0 on success @@ -82,15 +82,17 @@ * as the second MSG arg struct for * OPTEE_FFA_YIELDING_CALL_WITH_ARG. * Bit[31:8]: Reserved (MBZ) - * w5: Bitfield of secure world capabilities OPTEE_FFA_SEC_CAP_* below, - * unused bits MBZ. - * w6-w7: Not used (MBZ) + * w5: Bitfield of OP-TEE capabilities OPTEE_FFA_SEC_CAP_* + * w6: The maximum secure world notification number + * w7: Not used (MBZ) */ /* * Secure world supports using an offset into the argument shared memory * object, see also OPTEE_FFA_YIELDING_CALL_WITH_ARG */ #define OPTEE_FFA_SEC_CAP_ARG_OFFSET BIT(0) +/* OP-TEE supports asynchronous notification via FF-A */ +#define OPTEE_FFA_SEC_CAP_ASYNC_NOTIF BIT(1) #define OPTEE_FFA_EXCHANGE_CAPABILITIES OPTEE_FFA_BLOCKING_CALL(2) @@ -109,6 +111,21 @@ */ #define OPTEE_FFA_UNREGISTER_SHM OPTEE_FFA_BLOCKING_CALL(3) +/* + * Inform OP-TEE that normal world is able to receive asynchronous + * notifications. + * + * Call register usage: + * w3: Service ID, OPTEE_FFA_ENABLE_ASYNC_NOTIF + * w4: Notification value to request bottom half processing + * w5-w7: Not used (MBZ) + * + * Return register usage: + * w3: Error code, 0 on success + * w4-w7: Not used (MBZ) + */ +#define OPTEE_FFA_ENABLE_ASYNC_NOTIF OPTEE_FFA_BLOCKING_CALL(5) + /* * Call with struct optee_msg_arg as argument in the supplied shared memory * with a zero internal offset and normal cached memory attributes. diff --git a/core/arch/arm/kernel/thread_spmc.c b/core/arch/arm/kernel/thread_spmc.c index 33323487fd8..8ef8de8c0ab 100644 --- a/core/arch/arm/kernel/thread_spmc.c +++ b/core/arch/arm/kernel/thread_spmc.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BSD-2-Clause /* - * Copyright (c) 2020-2021, Linaro Limited. + * Copyright (c) 2020-2023, Linaro Limited. * Copyright (c) 2019-2021, Arm Limited. All rights reserved. */ @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -46,6 +47,11 @@ struct mem_frag_state { }; #endif +static unsigned int spmc_notif_lock = SPINLOCK_UNLOCK; +static int do_bottom_half_value = -1; +static uint16_t notif_vm_id; +static bool spmc_notif_is_ready; + /* Initialized in spmc_init() below */ static uint16_t my_endpoint_id __nex_bss; #ifdef CFG_CORE_SEL1_SPMC @@ -96,6 +102,11 @@ static bool is_nw_buf(struct ffa_rxtx *rxtx) static SLIST_HEAD(mem_frag_state_head, mem_frag_state) frag_state_head = SLIST_HEAD_INITIALIZER(&frag_state_head); + +static uint64_t notif_pending_bitmap; +static uint64_t notif_bound_bitmap; +static bool notif_vm_id_valid; +static int notif_intid = -1; #else static uint8_t __rx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE); static uint8_t __tx_buf[SMALL_PAGE_SIZE] __aligned(SMALL_PAGE_SIZE); @@ -146,10 +157,17 @@ uint32_t spmc_exchange_version(uint32_t vers, struct ffa_rxtx *rxtx) #if defined(CFG_CORE_SEL1_SPMC) static void handle_features(struct thread_smc_args *args) { - uint32_t ret_fid = 0; - uint32_t ret_w2 = FFA_PARAM_MBZ; + uint32_t ret_fid = FFA_ERROR; + uint32_t ret_w2 = FFA_NOT_SUPPORTED; switch (args->a1) { + case FFA_FEATURE_SCHEDULE_RECV_INTR: + if (spmc_notif_is_ready) { + ret_fid = FFA_SUCCESS_32; + ret_w2 = notif_intid; + } + break; + #ifdef ARM64 case FFA_RXTX_MAP_64: #endif @@ -187,11 +205,20 @@ static void handle_features(struct thread_smc_args *args) case FFA_RXTX_UNMAP: case FFA_RX_RELEASE: case FFA_FEATURE_MANAGED_EXIT_INTR: + case FFA_NOTIFICATION_BITMAP_CREATE: + case FFA_NOTIFICATION_BITMAP_DESTROY: + case FFA_NOTIFICATION_BIND: + case FFA_NOTIFICATION_UNBIND: + case FFA_NOTIFICATION_SET: + case FFA_NOTIFICATION_GET: + case FFA_NOTIFICATION_INFO_GET_32: +#ifdef ARM64 + case FFA_NOTIFICATION_INFO_GET_64: +#endif ret_fid = FFA_SUCCESS_32; + ret_w2 = FFA_PARAM_MBZ; break; default: - ret_fid = FFA_ERROR; - ret_w2 = FFA_NOT_SUPPORTED; break; } @@ -582,6 +609,30 @@ static void spmc_handle_run(struct thread_smc_args *args) } #endif /*CFG_CORE_SEL1_SPMC*/ +static uint32_t spmc_enable_async_notif(uint32_t bottom_half_value, + uint16_t vm_id) +{ + uint32_t old_itr_status = 0; + + if (!spmc_notif_is_ready) { + /* + * This should never happen, not if normal world respects the + * exchanged capabilities. + */ + EMSG("Asynchronous notifications are not ready"); + return TEE_ERROR_NOT_IMPLEMENTED; + } + + old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); + do_bottom_half_value = bottom_half_value; + if (!IS_ENABLED(CFG_CORE_SEL1_SPMC)) + notif_vm_id = vm_id; + cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); + + notif_deliver_atomic_event(NOTIF_EVENT_STARTED); + return TEE_SUCCESS; +} + static void handle_yielding_call(struct thread_smc_args *args, uint32_t direct_resp_fid) { @@ -630,6 +681,8 @@ static uint32_t handle_unregister_shm(uint32_t a4, uint32_t a5) static void handle_blocking_call(struct thread_smc_args *args, uint32_t direct_resp_fid) { + uint32_t sec_caps = 0; + switch (args->a3) { case OPTEE_FFA_GET_API_VERSION: spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, @@ -642,14 +695,24 @@ static void handle_blocking_call(struct thread_smc_args *args, CFG_OPTEE_REVISION_MINOR, TEE_IMPL_GIT_SHA1); break; case OPTEE_FFA_EXCHANGE_CAPABILITIES: - spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, - 0, THREAD_RPC_MAX_NUM_PARAMS, - OPTEE_FFA_SEC_CAP_ARG_OFFSET); + sec_caps = OPTEE_FFA_SEC_CAP_ARG_OFFSET; + if (spmc_notif_is_ready) + sec_caps |= OPTEE_FFA_SEC_CAP_ASYNC_NOTIF; + spmc_set_args(args, direct_resp_fid, + swap_src_dst(args->a1), 0, 0, + THREAD_RPC_MAX_NUM_PARAMS, sec_caps); break; case OPTEE_FFA_UNREGISTER_SHM: spmc_set_args(args, direct_resp_fid, swap_src_dst(args->a1), 0, handle_unregister_shm(args->a4, args->a5), 0, 0); break; + case OPTEE_FFA_ENABLE_ASYNC_NOTIF: + spmc_set_args(args, direct_resp_fid, + swap_src_dst(args->a1), 0, + spmc_enable_async_notif(args->a4, + FFA_SRC(args->a1)), + 0, 0); + break; default: EMSG("Unhandled blocking service ID %#"PRIx32, (uint32_t)args->a3); @@ -1304,6 +1367,222 @@ static void handle_mem_reclaim(struct thread_smc_args *args) out: spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); } + +static void handle_notification_bitmap_create(struct thread_smc_args *args) +{ + uint32_t ret_val = FFA_INVALID_PARAMETERS; + uint32_t ret_fid = FFA_ERROR; + uint32_t old_itr_status = 0; + + if (!FFA_TARGET_INFO_GET_SP_ID(args->a1) && !args->a3 && !args->a4 && + !args->a5 && !args->a6 && !args->a7) { + uint16_t vm_id = args->a1; + + old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); + + if (notif_vm_id_valid) { + if (vm_id == notif_vm_id) + ret_val = FFA_DENIED; + else + ret_val = FFA_NO_MEMORY; + } else { + notif_vm_id = vm_id; + notif_vm_id_valid = true; + ret_val = FFA_OK; + ret_fid = FFA_SUCCESS_32; + } + + cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); + } + + spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); +} + +static void handle_notification_bitmap_destroy(struct thread_smc_args *args) +{ + uint32_t ret_val = FFA_INVALID_PARAMETERS; + uint32_t ret_fid = FFA_ERROR; + uint32_t old_itr_status = 0; + + if (!FFA_TARGET_INFO_GET_SP_ID(args->a1) && !args->a3 && !args->a4 && + !args->a5 && !args->a6 && !args->a7) { + uint16_t vm_id = args->a1; + + old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); + + if (notif_vm_id_valid && vm_id == notif_vm_id) { + if (notif_pending_bitmap || notif_bound_bitmap) { + ret_val = FFA_DENIED; + } else { + notif_vm_id_valid = false; + ret_val = FFA_OK; + ret_fid = FFA_SUCCESS_32; + } + } + + cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); + } + + spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); +} + +static void handle_notification_bind(struct thread_smc_args *args) +{ + uint32_t ret_val = FFA_INVALID_PARAMETERS; + uint32_t ret_fid = FFA_ERROR; + uint32_t old_itr_status = 0; + uint64_t bitmap = 0; + uint16_t vm_id = 0; + + if (args->a5 || args->a6 || args->a7) + goto out; + if (args->a2) { + /* We only deal with global notifications for now */ + ret_val = FFA_NOT_SUPPORTED; + goto out; + } + + /* The destination of the eventual notification */ + vm_id = FFA_DST(args->a1); + bitmap = reg_pair_to_64(args->a4, args->a3); + + old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); + + if (notif_vm_id_valid && vm_id == notif_vm_id) { + if (bitmap & notif_bound_bitmap) { + ret_val = FFA_DENIED; + } else { + notif_bound_bitmap |= bitmap; + ret_val = FFA_OK; + ret_fid = FFA_SUCCESS_32; + } + } + + cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); +out: + spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); +} + +static void handle_notification_unbind(struct thread_smc_args *args) +{ + uint32_t ret_val = FFA_INVALID_PARAMETERS; + uint32_t ret_fid = FFA_ERROR; + uint32_t old_itr_status = 0; + uint64_t bitmap = 0; + uint16_t vm_id = 0; + + if (args->a2 || args->a5 || args->a6 || args->a7) + goto out; + + /* The destination of the eventual notification */ + vm_id = FFA_DST(args->a1); + bitmap = reg_pair_to_64(args->a4, args->a3); + + old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); + + if (notif_vm_id_valid && vm_id == notif_vm_id) { + /* + * Spec says: + * At least one notification is bound to another Sender or + * is currently pending. + * + * Not sure what the intention is. + */ + if (bitmap & notif_pending_bitmap) { + ret_val = FFA_DENIED; + } else { + notif_bound_bitmap &= ~bitmap; + ret_val = FFA_OK; + ret_fid = FFA_SUCCESS_32; + } + } + + cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); +out: + spmc_set_args(args, ret_fid, 0, ret_val, 0, 0, 0); +} + +static void handle_notification_get(struct thread_smc_args *args) +{ + uint32_t w2 = FFA_INVALID_PARAMETERS; + uint32_t ret_fid = FFA_ERROR; + uint32_t old_itr_status = 0; + uint16_t vm_id = 0; + uint32_t w3 = 0; + + if (args->a5 || args->a6 || args->a7) + goto out; + if (!(args->a2 & 0x1)) { + ret_fid = FFA_SUCCESS_32; + w2 = 0; + goto out; + } + vm_id = FFA_SRC(args->a1); + + old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); + + if (notif_vm_id_valid && vm_id == notif_vm_id) { + reg_pair_from_64(notif_pending_bitmap, &w3, &w2); + notif_pending_bitmap = 0; + ret_fid = FFA_SUCCESS_32; + } + + cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); +out: + spmc_set_args(args, ret_fid, 0, w2, w3, 0, 0); +} + +static void handle_notification_info_get(struct thread_smc_args *args) +{ + uint32_t w2 = FFA_INVALID_PARAMETERS; + uint32_t ret_fid = FFA_ERROR; + + if (args->a1 || args->a2 || args->a3 || args->a4 || args->a5 || + args->a6 || args->a7) + goto out; + + if (OPTEE_SMC_IS_64(args->a0)) + ret_fid = FFA_SUCCESS_64; + else + ret_fid = FFA_SUCCESS_32; + + /* + * Note, we're only supporting physical OS kernel in normal world + * with Global Notifications. + * So one list of ID list registers (BIT[11:7]) + * and one count of IDs (BIT[13:12] + 1) + * and the VM is always 0. + */ + w2 = SHIFT_U32(1, 7); +out: + spmc_set_args(args, ret_fid, 0, w2, 0, 0, 0); +} + +void thread_spmc_set_async_notif_intid(int intid) +{ + assert(interrupt_can_raise_sgi(interrupt_get_main_chip())); + notif_intid = intid; + spmc_notif_is_ready = true; + DMSG("Asynchronous notifications are ready"); +} + +void notif_send_async(uint32_t value) +{ + uint32_t old_itr_status = 0; + + old_itr_status = cpu_spin_lock_xsave(&spmc_notif_lock); + assert(value == NOTIF_VALUE_DO_BOTTOM_HALF && spmc_notif_is_ready && + do_bottom_half_value >= 0 && notif_intid >= 0); + notif_pending_bitmap |= BIT64(do_bottom_half_value); + interrupt_raise_sgi(interrupt_get_main_chip(), notif_intid, + ITR_CPU_MASK_TO_THIS_CPU); + cpu_spin_unlock_xrestore(&spmc_notif_lock, old_itr_status); +} +#else +void __noreturn notif_send_async(uint32_t value __unused) +{ + panic(); +} #endif /* Only called from assembly */ @@ -1367,7 +1646,40 @@ void thread_spmc_msg_recv(struct thread_smc_args *args) case FFA_MEM_FRAG_TX: handle_mem_frag_tx(args, &my_rxtx); break; + case FFA_NOTIFICATION_BITMAP_CREATE: + handle_notification_bitmap_create(args); + break; + case FFA_NOTIFICATION_BITMAP_DESTROY: + handle_notification_bitmap_destroy(args); + break; + case FFA_NOTIFICATION_BIND: + handle_notification_bind(args); + break; + case FFA_NOTIFICATION_UNBIND: + handle_notification_unbind(args); + break; + case FFA_NOTIFICATION_GET: + handle_notification_get(args); + break; +#ifdef ARM64 + case FFA_NOTIFICATION_INFO_GET_64: +#endif + case FFA_NOTIFICATION_INFO_GET_32: + handle_notification_info_get(args); + break; #endif /*CFG_CORE_SEL1_SPMC*/ + case FFA_ERROR: + EMSG("Cannot handle FFA_ERROR(%d)", (int)args->a2); + if (!IS_ENABLED(CFG_CORE_SEL1_SPMC)) { + /* + * The SPMC will return an FFA_ERROR back so better + * panic() now than flooding the log. + */ + panic("FFA_ERROR from SPMC is fatal"); + } + spmc_set_args(args, FFA_ERROR, FFA_PARAM_MBZ, FFA_NOT_SUPPORTED, + FFA_PARAM_MBZ, FFA_PARAM_MBZ, FFA_PARAM_MBZ); + break; default: EMSG("Unhandled FFA function ID %#"PRIx32, (uint32_t)args->a0); spmc_set_args(args, FFA_ERROR, FFA_PARAM_MBZ, FFA_NOT_SUPPORTED, diff --git a/core/arch/arm/mm/mobj_ffa.c b/core/arch/arm/mm/mobj_ffa.c index e57c95ac83c..8d3d71447fe 100644 --- a/core/arch/arm/mm/mobj_ffa.c +++ b/core/arch/arm/mm/mobj_ffa.c @@ -423,7 +423,17 @@ TEE_Result mobj_ffa_unregister_by_cookie(uint64_t cookie) } #if defined(CFG_CORE_SEL1_SPMC) if (!mf->registered_by_cookie) { - EMSG("cookie %#"PRIx64" not registered", cookie); + /* + * This is expected behaviour if the normal world has + * registered the memory but OP-TEE has not yet used the + * corresponding cookie with mobj_ffa_get_by_cookie(). It + * can be non-trivial for the normal world to predict if + * the cookie really has been used or not. So even if we + * return it as an error it will be ignored by + * handle_unregister_shm(). + */ + EMSG("cookie %#"PRIx64" not registered refs %u:%u", + cookie, refcount_val(&mf->mobj.refc), mf->inactive_refs); res = TEE_ERROR_ITEM_NOT_FOUND; goto out; } diff --git a/core/arch/arm/plat-vexpress/conf.mk b/core/arch/arm/plat-vexpress/conf.mk index 452aba01590..eeb7eccf9be 100644 --- a/core/arch/arm/plat-vexpress/conf.mk +++ b/core/arch/arm/plat-vexpress/conf.mk @@ -148,6 +148,8 @@ CFG_DTB_MAX_SIZE ?= 0x100000 ifeq ($(CFG_SCMI_SCPFW),y) $(call force,CFG_SCMI_SCPFW_PRODUCT,optee-fvp) endif +CFG_CORE_ASYNC_NOTIF ?= y +CFG_CORE_ASYNC_NOTIF_GIC_INTID ?= 219 endif ifneq (,$(filter $(PLATFORM_FLAVOR),qemu_virt qemu_armv8a)) diff --git a/core/arch/arm/plat-vexpress/main.c b/core/arch/arm/plat-vexpress/main.c index 4aaf10f435e..e12660f7150 100644 --- a/core/arch/arm/plat-vexpress/main.c +++ b/core/arch/arm/plat-vexpress/main.c @@ -1,10 +1,11 @@ // SPDX-License-Identifier: BSD-2-Clause /* - * Copyright (c) 2016-2020, Linaro Limited + * Copyright (c) 2016-2023, Linaro Limited * Copyright (c) 2014, STMicroelectronics International N.V. */ #include +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -44,20 +46,31 @@ register_ddr(DRAM1_BASE, DRAM1_SIZE); #endif #ifdef CFG_GIC +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICC_BASE, GIC_CPU_REG_SIZE); register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICD_BASE, GIC_DIST_REG_SIZE); -register_phys_mem_pgdir(MEM_AREA_IO_SEC, GICC_BASE, GIC_DIST_REG_SIZE); +#ifdef GIC_REDIST_BASE +register_phys_mem_pgdir(MEM_AREA_IO_SEC, GIC_REDIST_BASE, GIC_REDIST_SIZE); +#endif void boot_primary_init_intc(void) { +#ifdef GIC_REDIST_BASE + gic_init_v3(GIC_BASE + GICC_OFFSET, GIC_BASE + GICD_OFFSET, + GIC_REDIST_BASE); +#else gic_init(GIC_BASE + GICC_OFFSET, GIC_BASE + GICD_OFFSET); +#endif + if (IS_ENABLED(CFG_CORE_SEL1_SPMC) && + IS_ENABLED(CFG_CORE_ASYNC_NOTIF)) { + gic_init_donate_sgi_to_ns(GIC_SGI_SEC_BASE); + thread_spmc_set_async_notif_intid(GIC_SGI_SEC_BASE); + } } -#if !defined(CFG_WITH_ARM_TRUSTED_FW) void boot_secondary_init_intc(void) { - gic_cpu_init(); + gic_init_per_cpu(); } -#endif #endif /*CFG_GIC*/ #ifdef CFG_CORE_HAFNIUM_INTC diff --git a/core/arch/arm/plat-vexpress/platform_config.h b/core/arch/arm/plat-vexpress/platform_config.h index 206885c0f28..078662a13eb 100644 --- a/core/arch/arm/plat-vexpress/platform_config.h +++ b/core/arch/arm/plat-vexpress/platform_config.h @@ -126,15 +126,17 @@ #define GICD_OFFSET 0 #define GICC_OFFSET 0x10000 +#ifdef CFG_ARM_GICV3 +#define GIC_REDIST_BASE 0x080A0000 +#define GIC_REDIST_SIZE 0x00F60000 +#endif #else #error "Unknown platform flavor" #endif -#ifdef GIC_BASE #define GICD_BASE (GIC_BASE + GICD_OFFSET) #define GICC_BASE (GIC_BASE + GICC_OFFSET) -#endif #ifndef UART_BAUDRATE #define UART_BAUDRATE 115200 diff --git a/core/core.mk b/core/core.mk index b023e46ac43..71c4699860a 100644 --- a/core/core.mk +++ b/core/core.mk @@ -15,6 +15,7 @@ PLATFORM_$(PLATFORM) := y PLATFORM_FLAVOR_$(PLATFORM_FLAVOR) := y $(eval $(call cfg-depends-all,CFG_PAGED_USER_TA,CFG_WITH_PAGER CFG_WITH_USER_TA)) +_CFG_CORE_ASYNC_NOTIF_DEFAULT_IMPL ?= $(CFG_CORE_ASYNC_NOTIF) include core/crypto.mk ifeq ($(CFG_SCMI_SCPFW),y) diff --git a/core/drivers/gic.c b/core/drivers/gic.c index 04b4fc466c9..47fad685e84 100644 --- a/core/drivers/gic.c +++ b/core/drivers/gic.c @@ -1,26 +1,27 @@ // SPDX-License-Identifier: BSD-2-Clause /* - * Copyright (c) 2016-2017, Linaro Limited + * Copyright (c) 2016-2017, 2023 Linaro Limited * Copyright (c) 2014, STMicroelectronics International N.V. */ #include #include #include -#include #include +#include #include +#include #include #include #include #include +#include #include +#include #include #include -#include -#include -#include #include +#include /* Offsets from gic.gicc_base */ #define GICC_CTLR (0x000) @@ -30,7 +31,6 @@ #define GICC_CTLR_ENABLEGRP0 (1 << 0) #define GICC_CTLR_ENABLEGRP1 (1 << 1) -#define GICD_CTLR_ENABLEGRP1S (1 << 2) #define GICC_CTLR_FIQEN (1 << 3) /* Offsets from gic.gicd_base */ @@ -46,8 +46,39 @@ #define GICD_IGROUPMODR(n) (0xd00 + (n) * 4) #define GICD_SGIR (0xF00) -#define GICD_CTLR_ENABLEGRP0 (1 << 0) -#define GICD_CTLR_ENABLEGRP1 (1 << 1) +#ifdef CFG_ARM_GICV3 +#define GICD_PIDR2 (0xFFE8) +#else +/* Called ICPIDR2 in GICv2 specification */ +#define GICD_PIDR2 (0xFE8) +#endif + +#define GICD_CTLR_ENABLEGRP0 BIT32(0) +#define GICD_CTLR_ENABLEGRP1NS BIT32(1) +#define GICD_CTLR_ENABLEGRP1S BIT32(2) +#define GICD_CTLR_ARE_S BIT32(4) +#define GICD_CTLR_ARE_NS BIT32(5) + +/* Offsets from gic.gicr_base[core_pos] */ +#define GICR_V3_PCPUBASE_SIZE (2 * 64 * 1024) +#define GICR_SGI_BASE_OFFSET (64 * 1024) +#define GICR_CTLR (0x00) +#define GICR_TYPER (0x08) + +#define GICR_IGROUPR0 (GICR_SGI_BASE_OFFSET + 0x080) +#define GICR_IGRPMODR0 (GICR_SGI_BASE_OFFSET + 0xD00) +#define GICR_ICENABLER0 (GICR_SGI_BASE_OFFSET + 0x180) +#define GICR_ICPENDR0 (GICR_SGI_BASE_OFFSET + 0x280) + +#define GICR_TYPER_LAST BIT64(4) +#define GICR_TYPER_AFF3_SHIFT 56 +#define GICR_TYPER_AFF2_SHIFT 48 +#define GICR_TYPER_AFF1_SHIFT 40 +#define GICR_TYPER_AFF0_SHIFT 32 + +/* GICD IDR2 name differs on GICv3 and GICv2 but uses same bit map */ +#define GICD_PIDR2_ARCHREV_SHIFT 4 +#define GICD_PIDR2_ARCHREV_MASK 0xF /* Number of Private Peripheral Interrupt */ #define NUM_PPI 32 @@ -88,7 +119,12 @@ struct gic_data { vaddr_t gicc_base; vaddr_t gicd_base; +#if defined(CFG_ARM_GICV3) + vaddr_t gicr_base[CFG_TEE_CORE_NB_CORE]; +#endif size_t max_it; + uint32_t per_cpu_group_status; + uint32_t per_cpu_group_modifier; struct itr_chip chip; }; @@ -116,6 +152,15 @@ static const struct itr_ops gic_ops = { }; DECLARE_KEEP_PAGER(gic_ops); +static vaddr_t __maybe_unused get_gicr_base(struct gic_data *gd __maybe_unused) +{ +#if defined(CFG_ARM_GICV3) + return gd->gicr_base[get_core_pos()]; +#else + return 0; +#endif +} + static size_t probe_max_it(vaddr_t gicc_base __maybe_unused, vaddr_t gicd_base) { int i; @@ -159,24 +204,91 @@ static size_t probe_max_it(vaddr_t gicc_base __maybe_unused, vaddr_t gicd_base) return ret; } -void gic_cpu_init(void) +static void gicv3_sync_sgi_config(struct gic_data *gd) { - struct gic_data *gd = &gic_data; + vaddr_t gicr_base = get_gicr_base(gd); + bool need_sync = false; + uint32_t gmod0 = 0; + uint32_t grp0 = 0; + size_t n = 0; + + if (!gicr_base) + panic("GICR_BASE missing for affinity routing"); + + grp0 = io_read32(gicr_base + GICR_IGROUPR0); + gmod0 = io_read32(gicr_base + GICR_IGRPMODR0); + for (n = GIC_SGI_SEC_BASE; n <= GIC_SGI_SEC_MAX; n++) { + /* Ignore matching bits */ + if (!(BIT32(n) & (grp0 ^ gd->per_cpu_group_status)) && + !(BIT32(n) & (gmod0 ^ gd->per_cpu_group_modifier))) + continue; + /* + * SGI-n differs from primary CPU configuration, + * let's sync up. + */ + need_sync = true; + + /* Disable interrupt */ + io_write32(gicr_base + GICR_ICENABLER0, BIT32(n)); + + /* Make interrupt non-pending */ + io_write32(gicr_base + GICR_ICPENDR0, BIT32(n)); + + if (BIT32(n) & gd->per_cpu_group_status) + grp0 |= BIT32(n); + else + grp0 &= ~BIT32(n); + if (BIT32(n) & gd->per_cpu_group_modifier) + gmod0 |= BIT32(n); + else + gmod0 &= ~BIT32(n); + } -#if defined(CFG_ARM_GICV3) - assert(gd->gicd_base); -#else - assert(gd->gicd_base && gd->gicc_base); -#endif + if (need_sync) { + io_write32(gicr_base + GICR_IGROUPR0, grp0); + io_write32(gicr_base + GICR_IGRPMODR0, gmod0); + } +} - /* per-CPU interrupts config: - * ID0-ID7(SGI) for Non-secure interrupts - * ID8-ID15(SGI) for Secure interrupts. - * All PPI config as Non-secure interrupts. - */ - io_write32(gd->gicd_base + GICD_IGROUPR(0), 0xffff00ff); +static void gic_legacy_sync_sgi_config(struct gic_data *gd) +{ + bool need_sync = false; + uint32_t grp0 = 0; + size_t n = 0; + + grp0 = io_read32(gd->gicd_base + GICD_IGROUPR(0)); + for (n = GIC_SGI_SEC_BASE; n <= GIC_SGI_SEC_MAX; n++) { + /* Ignore matching bits */ + if (!(BIT32(n) & (grp0 ^ gd->per_cpu_group_status))) + continue; + /* + * SGI-n differs from primary CPU configuration, + * let's sync up. + */ + need_sync = true; + + /* Disable interrupt */ + io_write32(gd->gicd_base + GICD_ICENABLER(0), BIT(n)); + + /* Make interrupt non-pending */ + io_write32(gd->gicd_base + GICD_ICPENDR(0), BIT(n)); + + if (BIT32(n) & gd->per_cpu_group_status) + grp0 |= BIT32(n); + else + grp0 &= ~BIT32(n); + } - /* Set the priority mask to permit Non-secure interrupts, and to + if (need_sync) + io_write32(gd->gicd_base + GICD_IGROUPR(0), grp0); +} + +static void init_gic_per_cpu(struct gic_data *gd) +{ + io_write32(gd->gicd_base + GICD_IGROUPR(0), gd->per_cpu_group_status); + + /* + * Set the priority mask to permit Non-secure interrupts, and to * allow the Non-secure world to adjust the priority mask itself */ #if defined(CFG_ARM_GICV3) @@ -192,6 +304,89 @@ void gic_cpu_init(void) #endif } +void gic_init_per_cpu(void) +{ + struct gic_data *gd = &gic_data; + +#if defined(CFG_ARM_GICV3) + assert(gd->gicd_base); +#else + assert(gd->gicd_base && gd->gicc_base); +#endif + + if (IS_ENABLED(CFG_WITH_ARM_TRUSTED_FW)) { + /* + * GIC is already initialized by TF-A, we only need to + * handle eventual SGI configuration changes. + */ + if (IS_ENABLED(CFG_ARM_GICV3) && + io_read32(gd->gicd_base + GICD_CTLR) & GICD_CTLR_ARE_S) + gicv3_sync_sgi_config(gd); + else + gic_legacy_sync_sgi_config(gd); + } else { + /* + * Non-TF-A case where all CPU specific configuration + * of GIC must be done here. + */ + init_gic_per_cpu(gd); + } +} + +void gic_cpu_init(void) +{ + struct gic_data *gd = &gic_data; + +#if defined(CFG_ARM_GICV3) + assert(gd->gicd_base); +#else + assert(gd->gicd_base && gd->gicc_base); +#endif + IMSG("%s is deprecated, please use gic_init_per_cpu()", __func__); + + init_gic_per_cpu(gd); +} + +void gic_init_donate_sgi_to_ns(size_t it) +{ + struct gic_data *gd = &gic_data; + + assert(it >= GIC_SGI_SEC_BASE && it <= GIC_SGI_SEC_MAX); + + /* Assert it's secure to start with. */ + assert(!(gd->per_cpu_group_status & BIT32(it)) && + (gd->per_cpu_group_modifier & BIT32(it))); + + gd->per_cpu_group_modifier &= ~BIT32(it); + gd->per_cpu_group_status |= BIT32(it); + + if (IS_ENABLED(CFG_ARM_GICV3) && + (io_read32(gd->gicd_base + GICD_CTLR) & GICD_CTLR_ARE_S)) { + vaddr_t gicr_base = get_gicr_base(gd); + + /* Disable interrupt */ + io_write32(gicr_base + GICR_ICENABLER0, BIT32(it)); + + /* Make interrupt non-pending */ + io_write32(gicr_base + GICR_ICPENDR0, BIT32(it)); + + /* Make it to non-secure */ + io_write32(gicr_base + GICR_IGROUPR0, gd->per_cpu_group_status); + io_write32(gicr_base + GICR_IGRPMODR0, + gd->per_cpu_group_modifier); + } else { + /* Disable interrupt */ + io_write32(gd->gicd_base + GICD_ICENABLER(0), BIT(it)); + + /* Make interrupt non-pending */ + io_write32(gd->gicd_base + GICD_ICPENDR(0), BIT(it)); + + /* Make it to non-secure */ + io_write32(gd->gicd_base + GICD_IGROUPR(0), + gd->per_cpu_group_status); + } +} + static int gic_dt_get_irq(const uint32_t *properties, int count, uint32_t *type, uint32_t *prio) { @@ -222,11 +417,54 @@ static int gic_dt_get_irq(const uint32_t *properties, int count, uint32_t *type, return it_num; } -static void gic_init_base_addr(paddr_t gicc_base_pa, paddr_t gicd_base_pa) +static void __maybe_unused probe_redist_base_addrs(vaddr_t *gicr_base_addrs, + paddr_t gicr_base_pa) +{ + size_t sz = GICR_V3_PCPUBASE_SIZE; + paddr_t pa = gicr_base_pa; + size_t core_pos = 0; + uint64_t mt_bit = 0; + uint64_t mpidr = 0; + uint64_t tv = 0; + vaddr_t va = 0; + +#ifdef ARM64 + mt_bit = read_mpidr_el1() & MPIDR_MT_MASK; +#endif + do { + va = core_mmu_get_va(pa, MEM_AREA_IO_SEC, sz); + if (!va) + panic(); + tv = io_read64(va + GICR_TYPER); + + /* + * Extract an mpidr from the Type register to calculate the + * core position of this redistributer instance. + */ + mpidr = mt_bit; + mpidr |= SHIFT_U64((tv >> GICR_TYPER_AFF3_SHIFT) & + MPIDR_AFFLVL_MASK, MPIDR_AFF3_SHIFT); + mpidr |= (tv >> GICR_TYPER_AFF0_SHIFT) & + (MPIDR_AFF0_MASK | MPIDR_AFF1_MASK | MPIDR_AFF2_MASK); + core_pos = get_core_pos_mpidr(mpidr); + if (core_pos < CFG_TEE_CORE_NB_CORE) { + DMSG("GICR_BASE[%zu] at %#"PRIxVA, core_pos, va); + gicr_base_addrs[core_pos] = va; + } else { + EMSG("Skipping too large core_pos %zu from GICR_TYPER", + core_pos); + } + pa += sz; + } while (!(tv & GICR_TYPER_LAST)); +} + +static void gic_init_base_addr(paddr_t gicc_base_pa, paddr_t gicd_base_pa, + paddr_t gicr_base_pa __maybe_unused) { struct gic_data *gd = &gic_data; vaddr_t gicc_base = 0; vaddr_t gicd_base = 0; + uint32_t vers __maybe_unused = 0; assert(cpu_mmu_enabled()); @@ -235,7 +473,14 @@ static void gic_init_base_addr(paddr_t gicc_base_pa, paddr_t gicd_base_pa) if (!gicd_base) panic(); - if (!IS_ENABLED(CFG_ARM_GICV3)) { + vers = io_read32(gicd_base + GICD_PIDR2); + vers >>= GICD_PIDR2_ARCHREV_SHIFT; + vers &= GICD_PIDR2_ARCHREV_MASK; + + if (IS_ENABLED(CFG_ARM_GICV3)) { + assert(vers == 3); + } else { + assert(vers == 2); gicc_base = core_mmu_get_va(gicc_base_pa, MEM_AREA_IO_SEC, GIC_CPU_REG_SIZE); if (!gicc_base) @@ -245,21 +490,45 @@ static void gic_init_base_addr(paddr_t gicc_base_pa, paddr_t gicd_base_pa) gd->gicc_base = gicc_base; gd->gicd_base = gicd_base; gd->max_it = probe_max_it(gicc_base, gicd_base); +#if defined(CFG_ARM_GICV3) + probe_redist_base_addrs(gd->gicr_base, gicr_base_pa); +#endif gd->chip.ops = &gic_ops; if (IS_ENABLED(CFG_DT)) gd->chip.dt_get_irq = gic_dt_get_irq; } -void gic_init(paddr_t gicc_base_pa, paddr_t gicd_base_pa) +void gic_init_v3(paddr_t gicc_base_pa, paddr_t gicd_base_pa, + paddr_t gicr_base_pa) { struct gic_data __maybe_unused *gd = &gic_data; size_t __maybe_unused n = 0; - gic_init_base_addr(gicc_base_pa, gicd_base_pa); + gic_init_base_addr(gicc_base_pa, gicd_base_pa, gicr_base_pa); +#if defined(CFG_WITH_ARM_TRUSTED_FW) /* GIC configuration is initialized from TF-A when embedded */ -#ifndef CFG_WITH_ARM_TRUSTED_FW + if (io_read32(gd->gicd_base + GICD_CTLR) & GICD_CTLR_ARE_S) { + vaddr_t gicr_base = get_gicr_base(gd); + + if (!gicr_base) + panic("GICR_BASE missing for affinity routing"); + /* Secure affinity routing enabled */ + gd->per_cpu_group_status = io_read32(gicr_base + GICR_IGROUPR0); + gd->per_cpu_group_modifier = io_read32(gicr_base + + GICR_IGRPMODR0); + } else { + /* Legacy operation with secure affinity routing disabled */ + gd->per_cpu_group_status = io_read32(gd->gicd_base + + GICD_IGROUPR(0)); + gd->per_cpu_group_modifier = ~gd->per_cpu_group_status; + } +#else /*!CFG_WITH_ARM_TRUSTED_FW*/ + /* + * Without TF-A, GIC is always configured in for legacy operation + * with secure affinity routing disabled. + */ for (n = 0; n <= gd->max_it / NUM_INTS_PER_REG; n++) { /* Disable interrupts */ io_write32(gd->gicd_base + GICD_ICENABLER(n), 0xffffffff); @@ -274,7 +543,10 @@ void gic_init(paddr_t gicc_base_pa, paddr_t gicd_base_pa) * ID8-ID15(SGI) for Secure interrupts. * All PPI config as Non-secure interrupts. */ - io_write32(gd->gicd_base + GICD_IGROUPR(n), 0xffff00ff); + gd->per_cpu_group_status = 0xffff00ff; + gd->per_cpu_group_modifier = ~gd->per_cpu_group_status; + io_write32(gd->gicd_base + GICD_IGROUPR(n), + gd->per_cpu_group_status); } else { io_write32(gd->gicd_base + GICD_IGROUPR(n), 0xffffffff); } @@ -294,9 +566,9 @@ void gic_init(paddr_t gicc_base_pa, paddr_t gicd_base_pa) io_write32(gd->gicc_base + GICC_CTLR, GICC_CTLR_FIQEN | GICC_CTLR_ENABLEGRP0 | GICC_CTLR_ENABLEGRP1); io_setbits32(gd->gicd_base + GICD_CTLR, - GICD_CTLR_ENABLEGRP0 | GICD_CTLR_ENABLEGRP1); + GICD_CTLR_ENABLEGRP0 | GICD_CTLR_ENABLEGRP1NS); #endif -#endif /*CFG_WITH_ARM_TRUSTED_FW*/ +#endif /*!CFG_WITH_ARM_TRUSTED_FW*/ interrupt_main_init(&gic_data.chip); } @@ -414,7 +686,7 @@ static void assert_cpu_mask_is_valid(uint32_t cpu_mask) } static void gic_it_raise_sgi(struct gic_data *gd __maybe_unused, size_t it, - uint32_t cpu_mask, uint8_t group) + uint32_t cpu_mask, bool ns) { #if defined(CFG_ARM_GICV3) uint32_t mask_id = it & 0xf; @@ -449,13 +721,13 @@ static void gic_it_raise_sgi(struct gic_data *gd __maybe_unused, size_t it, } /* Raise the interrupt */ - if (group) + if (ns) write_icc_asgi1r(mask); else write_icc_sgi1r(mask); #else uint32_t mask_id = it & GICD_SGIR_SIGINTID_MASK; - uint32_t mask_group = group & 0x1; + uint32_t mask_group = ns; uint32_t mask = mask_id; assert_cpu_mask_is_valid(cpu_mask); @@ -631,19 +903,15 @@ static void gic_op_raise_sgi(struct itr_chip *chip, size_t it, uint32_t cpu_mask) { struct gic_data *gd = container_of(chip, struct gic_data, chip); + bool ns = false; assert(gd == &gic_data); /* Should be Software Generated Interrupt */ assert(it < NUM_SGI); - if (it > gd->max_it) - panic(); - - if (it < NUM_NS_SGI) - gic_it_raise_sgi(gd, it, cpu_mask, 1); - else - gic_it_raise_sgi(gd, it, cpu_mask, 0); + ns = BIT32(it) & gd->per_cpu_group_status; + gic_it_raise_sgi(gd, it, cpu_mask, ns); } static void gic_op_set_affinity(struct itr_chip *chip, size_t it, diff --git a/core/include/drivers/gic.h b/core/include/drivers/gic.h index cb580ec1cfd..de7e45aa911 100644 --- a/core/include/drivers/gic.h +++ b/core/include/drivers/gic.h @@ -25,14 +25,41 @@ #define GIC_SPI_TO_ITNUM(x) ((x) + GIC_SPI_BASE) /* - * The two gic_init_* functions initializes the struct gic_data which is - * then used by the other functions. + * Default lowest ID for secure SGIs, note that this does not account for + * interrupts donated to non-secure world with gic_init_donate_sgi_to_ns(). */ +#define GIC_SGI_SEC_BASE 8 +/* Max ID for secure SGIs */ +#define GIC_SGI_SEC_MAX 15 -/* Initialize GIC */ -void gic_init(paddr_t gicc_base_pa, paddr_t gicd_base_pa); +/* + * The two gic_init() and gic_init_v3() functions initializes the struct + * gic_data which is then used by the other functions. These two functions + * also initializes the GIC and are only supposed to be called from the + * primary boot CPU. + */ +void gic_init_v3(paddr_t gicc_base_pa, paddr_t gicd_base_pa, + paddr_t gicr_base_pa); +static inline void gic_init(paddr_t gicc_base_pa, paddr_t gicd_base_pa) +{ + gic_init_v3(gicc_base_pa, gicd_base_pa, 0); +} + +/* Donates one of the secure SGIs to normal world */ +void gic_init_donate_sgi_to_ns(size_t it); + +/* + * Does per-CPU specific GIC initialization, should be called by all + * secondary CPUs when booting. + */ +void gic_init_per_cpu(void); -/* Only initialize CPU GIC interface, mainly use for secondary CPUs */ +/* + * Only initialize CPU GIC interface, mainly use for secondary CPUs in + * non-TF-A configurations. + * + * This function is deprecated, please use gic_init_per_cpu() instead. + */ void gic_cpu_init(void); /* Print GIC state to console */ diff --git a/core/kernel/notif.c b/core/kernel/notif.c index abf9feafabf..d23b4fcd816 100644 --- a/core/kernel/notif.c +++ b/core/kernel/notif.c @@ -1,11 +1,8 @@ // SPDX-License-Identifier: BSD-2-Clause /* - * Copyright (c) 2021, Linaro Limited + * Copyright (c) 2021-2023, Linaro Limited */ -#include -#include -#include #include #include #include @@ -16,98 +13,12 @@ #if defined(CFG_CORE_ASYNC_NOTIF) static struct mutex notif_mutex = MUTEX_INITIALIZER; static unsigned int notif_lock = SPINLOCK_UNLOCK; +static bool notif_started; SLIST_HEAD(notif_driver_head, notif_driver); static struct notif_driver_head notif_driver_head = SLIST_HEAD_INITIALIZER(¬if_driver_head); -static bitstr_t bit_decl(notif_values, NOTIF_ASYNC_VALUE_MAX + 1); -static bitstr_t bit_decl(notif_alloc_values, NOTIF_ASYNC_VALUE_MAX + 1); -static bool notif_started; - -TEE_Result notif_alloc_async_value(uint32_t *val) -{ - static bool alloc_values_inited; - uint32_t old_itr_status = 0; - int bit = 0; - - assert(interrupt_can_raise_pi(interrupt_get_main_chip())); - - old_itr_status = cpu_spin_lock_xsave(¬if_lock); - - if (!alloc_values_inited) { - bit_set(notif_alloc_values, NOTIF_VALUE_DO_BOTTOM_HALF); - alloc_values_inited = true; - } - - bit_ffc(notif_alloc_values, (int)NOTIF_ASYNC_VALUE_MAX + 1, &bit); - if (bit >= 0) { - *val = bit; - bit_set(notif_alloc_values, bit); - } - - cpu_spin_unlock_xrestore(¬if_lock, old_itr_status); - - if (bit < 0) - return TEE_ERROR_OUT_OF_MEMORY; - - return TEE_SUCCESS; -} - -void notif_free_async_value(uint32_t val) -{ - uint32_t old_itr_status = 0; - - old_itr_status = cpu_spin_lock_xsave(¬if_lock); - - assert(val < NOTIF_ASYNC_VALUE_MAX); - assert(bit_test(notif_alloc_values, val)); - bit_clear(notif_alloc_values, val); - - cpu_spin_unlock_xrestore(¬if_lock, old_itr_status); -} - -uint32_t notif_get_value(bool *value_valid, bool *value_pending) -{ - uint32_t old_itr_status = 0; - uint32_t res = 0; - int bit = 0; - - old_itr_status = cpu_spin_lock_xsave(¬if_lock); - - bit_ffs(notif_values, (int)NOTIF_ASYNC_VALUE_MAX + 1, &bit); - *value_valid = (bit >= 0); - if (!*value_valid) { - *value_pending = false; - goto out; - } - - res = bit; - bit_clear(notif_values, res); - bit_ffs(notif_values, (int)NOTIF_ASYNC_VALUE_MAX + 1, &bit); - *value_pending = (bit >= 0); -out: - cpu_spin_unlock_xrestore(¬if_lock, old_itr_status); - - return res; -} - -void notif_send_async(uint32_t value) -{ - uint32_t old_itr_status = 0; - struct itr_chip *itr_chip = interrupt_get_main_chip(); - - static_assert(CFG_CORE_ASYNC_NOTIF_GIC_INTID >= GIC_PPI_BASE); - - assert(value <= NOTIF_ASYNC_VALUE_MAX); - old_itr_status = cpu_spin_lock_xsave(¬if_lock); - - DMSG("0x%"PRIx32, value); - bit_set(notif_values, value); - interrupt_raise_pi(itr_chip, CFG_CORE_ASYNC_NOTIF_GIC_INTID); - - cpu_spin_unlock_xrestore(¬if_lock, old_itr_status); -} bool notif_async_is_started(void) { @@ -125,8 +36,6 @@ void notif_register_driver(struct notif_driver *ndrv) { uint32_t old_itr_status = 0; - assert(interrupt_can_raise_pi(interrupt_get_main_chip())); - old_itr_status = cpu_spin_lock_xsave(¬if_lock); SLIST_INSERT_HEAD(¬if_driver_head, ndrv, link); diff --git a/core/kernel/notif_default.c b/core/kernel/notif_default.c new file mode 100644 index 00000000000..85b156867d6 --- /dev/null +++ b/core/kernel/notif_default.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * Copyright (c) 2021-2023, Linaro Limited + */ + +#include +#include +#include +#include +#include +#include +#include + +static bitstr_t bit_decl(notif_values, NOTIF_ASYNC_VALUE_MAX + 1); +static bitstr_t bit_decl(notif_alloc_values, NOTIF_ASYNC_VALUE_MAX + 1); +static unsigned int notif_default_lock = SPINLOCK_UNLOCK; + +TEE_Result notif_alloc_async_value(uint32_t *val) +{ + static bool alloc_values_inited; + uint32_t old_itr_status = 0; + int bit = 0; + + assert(interrupt_can_raise_pi(interrupt_get_main_chip())); + + old_itr_status = cpu_spin_lock_xsave(¬if_default_lock); + + if (!alloc_values_inited) { + bit_set(notif_alloc_values, NOTIF_VALUE_DO_BOTTOM_HALF); + alloc_values_inited = true; + } + + bit_ffc(notif_alloc_values, (int)NOTIF_ASYNC_VALUE_MAX + 1, &bit); + if (bit >= 0) { + *val = bit; + bit_set(notif_alloc_values, bit); + } + + cpu_spin_unlock_xrestore(¬if_default_lock, old_itr_status); + + if (bit < 0) + return TEE_ERROR_OUT_OF_MEMORY; + + return TEE_SUCCESS; +} + +void notif_free_async_value(uint32_t val) +{ + uint32_t old_itr_status = 0; + + old_itr_status = cpu_spin_lock_xsave(¬if_default_lock); + + assert(val < NOTIF_ASYNC_VALUE_MAX); + assert(bit_test(notif_alloc_values, val)); + bit_clear(notif_alloc_values, val); + + cpu_spin_unlock_xrestore(¬if_default_lock, old_itr_status); +} + +uint32_t notif_get_value(bool *value_valid, bool *value_pending) +{ + uint32_t old_itr_status = 0; + uint32_t res = 0; + int bit = 0; + + old_itr_status = cpu_spin_lock_xsave(¬if_default_lock); + + bit_ffs(notif_values, (int)NOTIF_ASYNC_VALUE_MAX + 1, &bit); + *value_valid = (bit >= 0); + if (!*value_valid) { + *value_pending = false; + goto out; + } + + res = bit; + bit_clear(notif_values, res); + bit_ffs(notif_values, (int)NOTIF_ASYNC_VALUE_MAX + 1, &bit); + *value_pending = (bit >= 0); +out: + cpu_spin_unlock_xrestore(¬if_default_lock, old_itr_status); + + return res; +} + +void notif_send_async(uint32_t value) +{ + uint32_t old_itr_status = 0; + struct itr_chip *itr_chip = interrupt_get_main_chip(); + + assert(value <= NOTIF_ASYNC_VALUE_MAX); + old_itr_status = cpu_spin_lock_xsave(¬if_default_lock); + + DMSG("0x%"PRIx32, value); + bit_set(notif_values, value); + interrupt_raise_pi(itr_chip, CFG_CORE_ASYNC_NOTIF_GIC_INTID); + + cpu_spin_unlock_xrestore(¬if_default_lock, old_itr_status); +} diff --git a/core/kernel/sub.mk b/core/kernel/sub.mk index 2d36985ef9b..3e81d5f8f12 100644 --- a/core/kernel/sub.mk +++ b/core/kernel/sub.mk @@ -39,6 +39,7 @@ srcs-y += mutex.c srcs-$(CFG_LOCKDEP) += mutex_lockdep.c srcs-y += wait_queue.c srcs-y += notif.c +srcs-$(_CFG_CORE_ASYNC_NOTIF_DEFAULT_IMPL) += notif_default.c srcs-y += thread.c ifeq ($(CFG_WITH_USER_TA),y) diff --git a/mk/config.mk b/mk/config.mk index d48b40c59c1..04e402298ab 100644 --- a/mk/config.mk +++ b/mk/config.mk @@ -998,11 +998,19 @@ CFG_PAN ?= n $(eval $(call cfg-depends-one,CFG_PAN,CFG_ARM64_core CFG_RV64_core CFG_RV32_core)) +ifeq ($(filter y, $(CFG_CORE_SEL1_SPMC) $(CFG_CORE_SEL2_SPMC) \ + $(CFG_CORE_EL3_SPMC)),y) +# FF-A case, handled via the FF-A ABI +CFG_CORE_ASYNC_NOTIF ?= y +$(call force,_CFG_CORE_ASYNC_NOTIF_DEFAULT_IMPL,n) +else # CFG_CORE_ASYNC_NOTIF is defined by the platform to enable support -# for sending asynchronous notifications to normal world. Note that an -# interrupt ID must be configurged by the platform too. Currently is only +# for sending asynchronous notifications to normal world. +# Interrupt ID must be configurged by the platform too. Currently is only # CFG_CORE_ASYNC_NOTIF_GIC_INTID defined. CFG_CORE_ASYNC_NOTIF ?= n +$(call force,_CFG_CORE_ASYNC_NOTIF_DEFAULT_IMPL,$(CFG_CORE_ASYNC_NOTIF)) +endif $(eval $(call cfg-enable-all-depends,CFG_MEMPOOL_REPORT_LAST_OFFSET, \ CFG_WITH_STATS))