diff --git a/core/arch/arm/kernel/thread.c b/core/arch/arm/kernel/thread.c index 4487ef026df..3748d5bfeac 100644 --- a/core/arch/arm/kernel/thread.c +++ b/core/arch/arm/kernel/thread.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include diff --git a/core/arch/arm/plat-stm32mp1/plat_tzc400.c b/core/arch/arm/plat-stm32mp1/plat_tzc400.c index 55f6d41d5a3..b65f6837874 100644 --- a/core/arch/arm/plat-stm32mp1/plat_tzc400.c +++ b/core/arch/arm/plat-stm32mp1/plat_tzc400.c @@ -71,6 +71,7 @@ static bool tzc_region_is_secure(unsigned int i, vaddr_t base, size_t size) static TEE_Result init_stm32mp1_tzc(void) { + TEE_Result res = TEE_ERROR_GENERIC; void *base = phys_to_virt(TZC_BASE, MEM_AREA_IO_SEC, 1); unsigned int region_index = 1; const uint64_t dram_start = DDR_BASE; @@ -108,8 +109,12 @@ static TEE_Result init_stm32mp1_tzc(void) panic("Unexpected TZC area on non-secure region"); } - itr_add(&tzc_itr_handler); - itr_enable(tzc_itr_handler.it); + res = interrupt_add_handler_with_chip(interrupt_get_main_chip(), + &tzc_itr_handler); + if (res) + panic(); + + interrupt_enable(tzc_itr_handler.chip, tzc_itr_handler.it); tzc_set_action(TZC_ACTION_INT); return TEE_SUCCESS; diff --git a/core/arch/arm/plat-stm32mp1/pm/psci.c b/core/arch/arm/plat-stm32mp1/pm/psci.c index f39c032b2f4..fcdd3fd58de 100644 --- a/core/arch/arm/plat-stm32mp1/pm/psci.c +++ b/core/arch/arm/plat-stm32mp1/pm/psci.c @@ -132,6 +132,8 @@ static void raise_sgi0_as_secure(void) static void release_secondary_early_hpen(size_t __unused pos) { + struct itr_chip *itr_chip = interrupt_get_main_chip(); + /* Need to send SIG#0 over Group0 after individual core 1 reset */ raise_sgi0_as_secure(); udelay(20); @@ -142,7 +144,7 @@ static void release_secondary_early_hpen(size_t __unused pos) BOOT_API_A7_CORE1_MAGIC_NUMBER); dsb_ishst(); - itr_raise_sgi(GIC_SEC_SGI_0, TARGET_CPU1_GIC_MASK); + itr_chip->ops->raise_sgi(itr_chip, GIC_SEC_SGI_0, TARGET_CPU1_GIC_MASK); } /* Override default psci_cpu_on() with platform specific sequence */ diff --git a/core/arch/arm/plat-synquacer/main.c b/core/arch/arm/plat-synquacer/main.c index a0f10f3ebd4..1e3f15b8dee 100644 --- a/core/arch/arm/plat-synquacer/main.c +++ b/core/arch/arm/plat-synquacer/main.c @@ -59,8 +59,11 @@ static struct itr_handler timer_itr = { static TEE_Result init_timer_itr(void) { - itr_add(&timer_itr); - itr_enable(IT_SEC_TIMER); + if (interrupt_add_handler_with_chip(interrupt_get_main_chip(), + &timer_itr)) + panic(); + + interrupt_enable(timer_itr.chip, timer_itr.it); /* Enable timer FIQ to fetch entropy required during boot */ generic_timer_start(TIMER_PERIOD_MS); diff --git a/core/arch/arm/plat-vexpress/main.c b/core/arch/arm/plat-vexpress/main.c index d111bdc7a4e..4236924d8d4 100644 --- a/core/arch/arm/plat-vexpress/main.c +++ b/core/arch/arm/plat-vexpress/main.c @@ -101,14 +101,14 @@ static void read_console(void) } } -static enum itr_return console_itr_cb(struct itr_handler *h __maybe_unused) +static enum itr_return console_itr_cb(struct itr_handler *hdl) { if (notif_async_is_started()) { /* * Asynchronous notifications are enabled, lets read from * uart in the bottom half instead. */ - itr_disable(IT_CONSOLE_UART); + interrupt_disable(hdl->chip, hdl->it); notif_send_async(NOTIF_VALUE_DO_BOTTOM_HALF); } else { read_console(); @@ -136,11 +136,11 @@ static void yielding_console_notif(struct notif_driver *ndrv __unused, switch (ev) { case NOTIF_EVENT_DO_BOTTOM_HALF: read_console(); - itr_enable(IT_CONSOLE_UART); + interrupt_enable(console_itr.chip, console_itr.it); break; case NOTIF_EVENT_STOPPED: DMSG("Asynchronous notifications stopped"); - itr_enable(IT_CONSOLE_UART); + interrupt_enable(console_itr.chip, console_itr.it); break; default: EMSG("Unknown event %d", (int)ev); @@ -154,8 +154,15 @@ struct notif_driver console_notif = { static TEE_Result init_console_itr(void) { - itr_add(&console_itr); - itr_enable(IT_CONSOLE_UART); + TEE_Result res = TEE_ERROR_GENERIC; + + res = interrupt_add_handler_with_chip(interrupt_get_main_chip(), + &console_itr); + if (res) + return res; + + interrupt_enable(console_itr.chip, console_itr.it); + if (IS_ENABLED(CFG_CORE_ASYNC_NOTIF)) notif_register_driver(&console_notif); return TEE_SUCCESS; diff --git a/core/drivers/atmel_piobu.c b/core/drivers/atmel_piobu.c index f060a1ebce3..ff582332576 100644 --- a/core/drivers/atmel_piobu.c +++ b/core/drivers/atmel_piobu.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -265,8 +266,16 @@ static struct itr_handler secumod_itr_handler = { static void secumod_interrupt_init(void) { - itr_add_type_prio(&secumod_itr_handler, IRQ_TYPE_LEVEL_HIGH, 7); - itr_enable(secumod_itr_handler.it); + TEE_Result res = TEE_ERROR_GENERIC; + + secumod_itr_handler.chip = interrupt_get_main_chip(); + + res = interrupt_add_configure_handler(&secumod_itr_handler, + IRQ_TYPE_LEVEL_HIGH, 7); + if (res) + panic(); + + interrupt_enable(secumod_itr_handler.chip, secumod_itr_handler.it); } static void secumod_cfg_input_pio(uint8_t gpio_pin, uint32_t config) diff --git a/core/drivers/atmel_saic.c b/core/drivers/atmel_saic.c index b8f5212ec47..adf2af7fa78 100644 --- a/core/drivers/atmel_saic.c +++ b/core/drivers/atmel_saic.c @@ -29,7 +29,7 @@ struct saic_data { uint32_t external[SAMA5D2_AIC_MAX_IRQS32]; }; -static struct saic_data saic = {0}; +static struct saic_data saic; static void saic_register_pm(void); @@ -47,7 +47,7 @@ void interrupt_main_handler(void) { uint32_t irqnr = saic_read_reg(AT91_AIC_IVR); - itr_handle(irqnr); + interrupt_call_handlers(&saic.chip, irqnr); saic_write_reg(AT91_AIC_EOICR, 0); } @@ -149,6 +149,8 @@ static void saic_set_affinity(struct itr_chip *chip __unused, static const struct itr_ops saic_ops = { .add = saic_add, + .mask = saic_disable, + .unmask = saic_enable, .enable = saic_enable, .disable = saic_disable, .raise_pi = saic_raise_pi, @@ -190,9 +192,11 @@ static int saic_dt_get_irq(const uint32_t *properties, int len, return it; } -struct itr_chip saic_chip = { - .ops = &saic_ops, - .dt_get_irq = &saic_dt_get_irq, +static struct saic_data saic = { + .chip = { + .ops = &saic_ops, + .dt_get_irq = &saic_dt_get_irq, + }, }; static void saic_clear_aicredir(void) @@ -295,7 +299,7 @@ TEE_Result atmel_saic_setup(void) saic_init_external(fdt, node); saic_init_hw(); - interrupt_main_init(&saic_chip); + interrupt_main_init(&saic.chip); saic_register_pm(); return TEE_SUCCESS; diff --git a/core/drivers/atmel_wdt.c b/core/drivers/atmel_wdt.c index b2aa5458d17..d35374ccfbb 100644 --- a/core/drivers/atmel_wdt.c +++ b/core/drivers/atmel_wdt.c @@ -11,8 +11,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -229,7 +231,8 @@ static TEE_Result wdt_node_probe(const void *fdt, int node, uint32_t irq_type = 0; uint32_t irq_prio = 0; int it = DT_INFO_INVALID_INTERRUPT; - struct itr_handler *it_hdlr; + struct itr_handler *it_hdlr = NULL; + TEE_Result res = TEE_ERROR_GENERIC; if (fdt_get_status(fdt, node) != DT_STATUS_OK_SEC) return TEE_ERROR_BAD_PARAMETERS; @@ -240,33 +243,50 @@ static TEE_Result wdt_node_probe(const void *fdt, int node, if (!wdt) return TEE_ERROR_OUT_OF_MEMORY; + it_hdlr = calloc(1, sizeof(*it_hdlr)); + if (!it_hdlr) { + free(wdt); + return TEE_ERROR_OUT_OF_MEMORY; + } + wdt->chip.ops = &atmel_wdt_ops; it = dt_get_irq_type_prio(fdt, node, &irq_type, &irq_prio); if (it == DT_INFO_INVALID_INTERRUPT) - goto err_free_wdt; + goto err_free; + + *it_hdlr = ITR_HANDLER(interrupt_get_main_chip(), it, 0, + atmel_wdt_itr_cb, wdt); - it_hdlr = itr_alloc_add_type_prio(it, &atmel_wdt_itr_cb, 0, wdt, - irq_type, irq_prio); - if (!it_hdlr) - goto err_free_wdt; + res = interrupt_add_configure_handler(it_hdlr, irq_type, irq_prio); + if (res) + goto err_free; if (dt_map_dev(fdt, node, &wdt->base, &size, DT_MAP_AUTO) < 0) - goto err_free_itr_handler; + goto err_remove_handler; /* Get current state of the watchdog */ wdt->mr = io_read32(wdt->base + WDT_MR) & WDT_MR_WDDIS; atmel_wdt_init_hw(wdt); - itr_enable(it); + interrupt_enable(it_hdlr->chip, it_hdlr->it); + + res = watchdog_register(&wdt->chip); + if (res) + goto err_disable_unmap; + atmel_wdt_register_pm(wdt); - return watchdog_register(&wdt->chip); + return TEE_SUCCESS; -err_free_itr_handler: - itr_free(it_hdlr); -err_free_wdt: +err_disable_unmap: + interrupt_disable(it_hdlr->chip, it_hdlr->it); + core_mmu_remove_mapping(MEM_AREA_IO_SEC, (void *)wdt->base, size); +err_remove_handler: + interrupt_remove_handler(it_hdlr); +err_free: free(wdt); + free(it_hdlr); return TEE_ERROR_GENERIC; } diff --git a/core/drivers/crypto/caam/caam_jr.c b/core/drivers/crypto/caam/caam_jr.c index eeb64f02f8f..a3bf68a329e 100644 --- a/core/drivers/crypto/caam/caam_jr.c +++ b/core/drivers/crypto/caam/caam_jr.c @@ -159,7 +159,7 @@ static enum caam_status do_jr_alloc(struct jr_privdata **privdata, static enum itr_return caam_jr_irqhandler(struct itr_handler *handler) { JR_TRACE("Disable the interrupt"); - itr_disable(handler->it); + interrupt_disable(handler->chip, handler->it); /* Send a signal to exit WFE loop */ sev(); @@ -582,13 +582,17 @@ enum caam_status caam_jr_init(struct caam_jrcfg *jrcfg) * Prepare the interrupt handler to secure the interrupt even * if the interrupt is not used */ + jr_privdata->it_handler.chip = interrupt_get_main_chip(); jr_privdata->it_handler.it = jrcfg->it_num; jr_privdata->it_handler.flags = ITRF_TRIGGER_LEVEL; jr_privdata->it_handler.handler = caam_jr_irqhandler; jr_privdata->it_handler.data = jr_privdata; #if defined(CFG_NXP_CAAM_RUNTIME_JR) && defined(CFG_CAAM_ITR) - itr_add(&jr_privdata->it_handler); + if (interrupt_add_handler(&jr_privdata->it_handler)) { + retstatus = CAAM_FAILURE; + goto end_init; + } #endif caam_hal_jr_enable_itr(jr_privdata->baseaddr); diff --git a/core/drivers/gic.c b/core/drivers/gic.c index 31884ab1457..cf40ed43474 100644 --- a/core/drivers/gic.c +++ b/core/drivers/gic.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -94,6 +95,8 @@ static void gic_op_set_affinity(struct itr_chip *chip, size_t it, static const struct itr_ops gic_ops = { .add = gic_op_add, + .mask = gic_op_disable, + .unmask = gic_op_enable, .enable = gic_op_enable, .disable = gic_op_disable, .raise_pi = gic_op_raise_pi, @@ -489,7 +492,7 @@ static void __maybe_unused gic_native_itr_handler(void) id = iar & GICC_IAR_IT_ID_MASK; if (id <= gd->max_it) - itr_handle(id); + interrupt_call_handlers(&gd->chip, id); else DMSG("ignoring interrupt %" PRIu32, id); @@ -585,3 +588,61 @@ static void gic_op_set_affinity(struct itr_chip *chip, size_t it, gic_it_set_cpu_mask(gd, it, cpu_mask); } + +#ifdef CFG_DT +/* Callback for "interrupt-extended" GIC interrupts in consumer DT nodes */ +static struct itr_desc *dt_get_gic_chip_cb(struct dt_pargs *arg, + void *priv_data, TEE_Result *res) +{ + int itr_num = DT_INFO_INVALID_INTERRUPT; + struct itr_chip *chip = priv_data; + struct itr_desc *desc = NULL; + uint32_t type = 0; + uint32_t prio = 0; + + itr_num = gic_dt_get_irq(arg->args, arg->args_count, &type, &prio); + if (itr_num == DT_INFO_INVALID_INTERRUPT) { + *res = TEE_ERROR_GENERIC; + return NULL; + } + + /* Allocate returned itr_desc as required by itr_dt_get_func type */ + desc = calloc(1, sizeof(*desc)); + if (!desc) { + *res = TEE_ERROR_OUT_OF_MEMORY; + return NULL; + } + + gic_op_add(chip, itr_num, type, prio); + + desc->chip = chip; + desc->itr_num = itr_num; + + *res = TEE_SUCCESS; + return desc; +} + +static TEE_Result gic_probe(const void *fdt, int offs, const void *cd __unused) +{ + if (interrupt_register_provider(fdt, offs, dt_get_gic_chip_cb, + &gic_data.chip)) + panic(); + + return TEE_SUCCESS; +} + +static const struct dt_device_match gic_match_table[] = { + { .compatible = "arm,cortex-a15-gic" }, + { .compatible = "arm,cortex-a7-gic" }, + { .compatible = "arm,cortex-a5-gic" }, + { .compatible = "arm,cortex-a9-gic" }, + { .compatible = "arm,gic-400" }, + { } +}; + +DEFINE_DT_DRIVER(gic_dt_driver) = { + .name = "gic", + .match_table = gic_match_table, + .probe = gic_probe, +}; +#endif /*CFG_DT*/ diff --git a/core/drivers/hfic.c b/core/drivers/hfic.c index 31c4b2e953c..cd308576a1e 100644 --- a/core/drivers/hfic.c +++ b/core/drivers/hfic.c @@ -60,6 +60,8 @@ static void hfic_op_set_affinity(struct itr_chip *chip __unused, static const struct itr_ops hfic_ops = { .add = hfic_op_add, + .mask = hfic_op_disable, + .unmask = hfic_op_enable, .enable = hfic_op_enable, .disable = hfic_op_disable, .raise_pi = hfic_op_raise_pi, @@ -85,7 +87,8 @@ void interrupt_main_handler(void) return; } - itr_handle(id); + interrupt_call_handlers(&hfic_data.chip, id); + res = thread_hvc(HF_INTERRUPT_DEACTIVATE, id, id, 0); assert(!res); } diff --git a/core/drivers/sp805_wdt.c b/core/drivers/sp805_wdt.c index 048620cde89..cd4624fea97 100644 --- a/core/drivers/sp805_wdt.c +++ b/core/drivers/sp805_wdt.c @@ -102,19 +102,21 @@ TEE_Result sp805_register_itr_handler(struct sp805_wdt_data *pd, uint32_t itr_num, uint32_t itr_flags, sp805_itr_handler_func_t itr_handler) { - struct itr_handler *wdt_itr; + TEE_Result res = TEE_ERROR_GENERIC; + struct itr_handler *wdt_itr = NULL; assert(!pd->chip.wdt_itr); - wdt_itr = itr_alloc_add(itr_num, wdt_itr_cb, - itr_flags, &pd->chip); - if (!wdt_itr) - return TEE_ERROR_OUT_OF_MEMORY; + res = interrupt_alloc_add_handler(interrupt_get_main_chip(), itr_num, + wdt_itr_cb, itr_flags, + &pd->chip, &wdt_itr); + if (res) + return res; pd->itr_handler = itr_handler; pd->chip.wdt_itr = wdt_itr; - itr_enable(wdt_itr->it); + interrupt_enable(wdt_itr->chip, wdt_itr->it); return TEE_SUCCESS; } diff --git a/core/include/kernel/dt.h b/core/include/kernel/dt.h index e6ca702a0b3..257fbd5b57e 100644 --- a/core/include/kernel/dt.h +++ b/core/include/kernel/dt.h @@ -7,7 +7,6 @@ #define KERNEL_DT_H #include -#include #include #include #include diff --git a/core/include/kernel/dt_driver.h b/core/include/kernel/dt_driver.h index d7f410974f0..e9e030c9306 100644 --- a/core/include/kernel/dt_driver.h +++ b/core/include/kernel/dt_driver.h @@ -23,6 +23,7 @@ * DT_DRIVER_I2C I2C bus controller using generic I2C bus DT bindings * DT_DRIVER_GPIO GPIO controller using generic GPIO DT bindings * DT_DRIVER_PINCTRL Pin controller using generic reset DT bindings + * DT_DRIVER_INTERRUPT Interrupt controller using generic DT bindings */ enum dt_driver_type { DT_DRIVER_NOTYPE, @@ -31,7 +32,8 @@ enum dt_driver_type { DT_DRIVER_RSTCTRL, DT_DRIVER_I2C, DT_DRIVER_GPIO, - DT_DRIVER_PINCTRL + DT_DRIVER_PINCTRL, + DT_DRIVER_INTERRUPT, }; /* @@ -169,6 +171,22 @@ void *dt_driver_device_from_node_idx_prop(const char *prop_name, void *dt_driver_device_from_parent(const void *fdt, int nodeoffset, enum dt_driver_type type, TEE_Result *res); +/* + * dt_driver_device_from_node_idx_prop_phandle() - Same as + * dt_driver_device_from_node_idx_prop() but phandle is not the first + * cells in property @prop_name but is passed as an argument. + * + * This function is used for DT bindings as "interrupts" property where the + * property carries the interrupt information but not the interrupt controller + * phandle which is found in a specific property (here "interrupt-parent"). + */ +void *dt_driver_device_from_node_idx_prop_phandle(const char *prop_name, + const void *fdt, int nodeoffs, + unsigned int prop_index, + enum dt_driver_type type, + uint32_t phandle, + TEE_Result *res); + /* * dt_driver_get_crypto() - Request crypto support for driver initialization * diff --git a/core/include/kernel/interrupt.h b/core/include/kernel/interrupt.h index 452b6fa4b28..ca92375dcd5 100644 --- a/core/include/kernel/interrupt.h +++ b/core/include/kernel/interrupt.h @@ -6,15 +6,29 @@ #define __KERNEL_INTERRUPT_H #include -#include +#include #include +#include +#include #include #define ITRF_TRIGGER_LEVEL BIT(0) -#define ITRF_SHARED BIT(1) +#define ITRF_SHARED BIT(1) + +struct itr_handler; +/* + * struct itr_chip - Interrupt controller + * + * @ops Operation callback functions + * @name Controller name, for debug purpose + * @handlers Registered handlers list head + * @dt_get_irq Device tree node parsing function + */ struct itr_chip { const struct itr_ops *ops; + const char *name; + SLIST_HEAD(, itr_handler) handlers; /* * dt_get_irq - parse a device tree interrupt property * @@ -28,11 +42,24 @@ struct itr_chip { uint32_t *prio); }; +/* + * struct itr_ops - Interrupt controller operations + * @add Register and configure an interrupt + * @enable Enable an interrupt + * @disable Disable an interrupt + * @mask Mask an interrupt, may be called from an interrupt context + * @unmask Unmask an interrupt, may be called from an interrupt context + * @raise_pi Raise per-cpu interrupt or NULL if not applicable + * @raise_sgi Raise a SGI or NULL if not applicable to that controller + * @set_affinity Set interrupt/cpu affinity or NULL if not applicable + */ struct itr_ops { void (*add)(struct itr_chip *chip, size_t it, uint32_t type, uint32_t prio); void (*enable)(struct itr_chip *chip, size_t it); void (*disable)(struct itr_chip *chip, size_t it); + void (*mask)(struct itr_chip *chip, size_t it); + void (*unmask)(struct itr_chip *chip, size_t it); void (*raise_pi)(struct itr_chip *chip, size_t it); void (*raise_sgi)(struct itr_chip *chip, size_t it, uint8_t cpu_mask); @@ -40,30 +67,76 @@ struct itr_ops { uint8_t cpu_mask); }; +/* + * struct itr_desc - Interrupt description + * @chip Interrupt controller reference + * @itr_num Interrupt number + * + * This struct is used for binding interrupt device data between + * drivers when using DT_DRIVERS means. See itr_dt_get_func type + * definition. + */ +struct itr_desc { + struct itr_chip *chip; + size_t itr_num; +}; + +/* Interrupt handler return value */ enum itr_return { ITRR_NONE, ITRR_HANDLED, }; -struct itr_handler; - +/* Interrupt handler signature */ typedef enum itr_return (*itr_handler_t)(struct itr_handler *h); +/* + * struct itr_handler - Interrupt handler reference + * @it Interrupt number + * @flags Property bit flags (ITRF_*) or 0 + * @data Private data for that interrupt handler + * @chip Interrupt controller chip device + * @link Reference in controller handler list + */ struct itr_handler { size_t it; uint32_t flags; itr_handler_t handler; void *data; + struct itr_chip *chip; SLIST_ENTRY(itr_handler) link; }; +#define ITR_HANDLER(_chip, _itr_num, _flags, _fn, _priv) \ + ((struct itr_handler){ \ + .chip = (_chip), .it = (_itr_num), .flags = (_flags), \ + .handler = (_fn), .data = (_priv), \ + }) + +/* + * Return true only if interrupt chip provides required handlers + * @chip: Interrupt controller reference + */ +static inline bool itr_chip_is_valid(struct itr_chip *chip) +{ + return chip && chip->ops && chip->ops->mask && chip->ops->unmask && + chip->ops->add; +} + /* - * Initialise core interrupt controller driver - * @data Core controller main data reference to register + * Initialise an interrupt controller handle + * @chip Interrupt controller + */ +TEE_Result itr_chip_init(struct itr_chip *chip); + +/* + * Initialise main interrupt controller driver + * @data Main controller main data reference to register */ void interrupt_main_init(struct itr_chip *data); -void itr_handle(size_t it); +/* Retrieve main interrupt controller reference */ +struct itr_chip *interrupt_get_main_chip(void); #ifdef CFG_DT /* @@ -92,45 +165,265 @@ static inline int dt_get_irq(const void *fdt, int node) } #endif -struct itr_handler *itr_alloc_add_type_prio(size_t it, itr_handler_t handler, - uint32_t flags, void *data, - uint32_t type, uint32_t prio); -void itr_free(struct itr_handler *hdl); -void itr_add_type_prio(struct itr_handler *handler, uint32_t type, - uint32_t prio); -void itr_enable(size_t it); -void itr_disable(size_t it); -/* raise the Peripheral Interrupt corresponding to the interrupt ID */ -void itr_raise_pi(size_t it); /* - * raise the Software Generated Interrupt corresponding to the interrupt ID, - * the cpu_mask represents which cpu interface to forward. + * __weak overridable function which is called when a secure interrupt is + * received. The default function calls panic() immediately, platforms which + * expects to receive secure interrupts should override this function. */ -void itr_raise_sgi(size_t it, uint8_t cpu_mask); +void interrupt_main_handler(void); + /* - * let corresponding interrupt forward to the cpu interface - * according to the cpu_mask. + * Interrupt controller chip API functions */ -void itr_set_affinity(size_t it, uint8_t cpu_mask); /* - * __weak overridable function which is called when a secure interrupt is - * received. The default function calls panic() immediately, platforms which - * expects to receive secure interrupts should override this function. + * interrupt_call_handlers() - Call registered handlers for an interrupt + * @chip Interrupt controller + * @itr_num Interrupt number + * + * This function is called from an interrupt context by a primary interrupt + * handler. This function calls the handlers registered for that interrupt. + * If interrupt is not handled, it is masked. */ -void interrupt_main_handler(void); +void interrupt_call_handlers(struct itr_chip *chip, size_t itr_num); + +/* + * interrupt_mask() - Mask an interrupt + * @chip Interrupt controller + * @itr_num Interrupt number + * This function may be called in interrupt context + */ +static inline void interrupt_mask(struct itr_chip *chip, size_t itr_num) +{ + chip->ops->mask(chip, itr_num); +} + +/* + * interrupt_unmask() - Unmask an interrupt + * @chip Interrupt controller + * @itr_num Interrupt number + * This function may be called in interrupt context + */ +static inline void interrupt_unmask(struct itr_chip *chip, size_t itr_num) +{ + chip->ops->unmask(chip, itr_num); +} + +/* + * interrupt_enable() - Enable an interrupt + * @chip Interrupt controller + * @itr_num Interrupt number + */ +static inline void interrupt_enable(struct itr_chip *chip, size_t itr_num) +{ + if (chip->ops->enable) + chip->ops->enable(chip, itr_num); + else + interrupt_unmask(chip, itr_num); +} + +/* + * interrupt_disable() - Disable an interrupt + * @chip Interrupt controller + * @itr_num Interrupt number + */ +static inline void interrupt_disable(struct itr_chip *chip, size_t itr_num) +{ + if (chip->ops->disable) + chip->ops->disable(chip, itr_num); + else + interrupt_mask(chip, itr_num); +} + +/* + * interrupt_configure() - Configure an interrupt in an interrupt controller + * @chip Interrupt controller + * @itr_num Interrupt number + * @type Trigger type ITR_TYPE_* of ITR_TYPE_NONE + * @prio Interrupt priority + * + * Interrupt consumer that get their interrupt from the DT do not need to + * call interrupt_configure() since the interrupt configuration has already + * been done by interrupt controller based on the DT bidings. + */ +TEE_Result interrupt_configure(struct itr_chip *chip, size_t itr_num, + uint32_t type, uint32_t prio); + +/* + * interrupt_add_and_configure_handler() - Register and configure a handler + * @hdl Interrupt handler to register + * @type Interrupt type (IRQ_TYPE_* defines) or IRQ_TYPE_NONE + * @prio Interrupt priority or 0 + */ +TEE_Result interrupt_add_configure_handler(struct itr_handler *hdl, + uint32_t type, uint32_t prio); -static inline void itr_add(struct itr_handler *handler) +/* + * interrupt_add_handler() - Register an interrupt handler + * @hdl Interrupt handler to register + * + * This helper function assumes interrypt type is set to IRQ_TYPE_NONE + * and interrupt priority to 0. + */ +static inline TEE_Result interrupt_add_handler(struct itr_handler *hdl) { - itr_add_type_prio(handler, IRQ_TYPE_NONE, 0); + return interrupt_add_configure_handler(hdl, IRQ_TYPE_NONE, 0); } -static inline struct itr_handler *itr_alloc_add(size_t it, - itr_handler_t handler, - uint32_t flags, void *data) +/* + * interrupt_add_handler_with_chip() - Register an interrupt handler providing + * the interrupt chip reference in specific argument @chip. + * + * @chip Interrupt controller + * @h Interrupt handler to register + */ +static inline TEE_Result interrupt_add_handler_with_chip(struct itr_chip *chip, + struct itr_handler *h) { - return itr_alloc_add_type_prio(it, handler, flags, data, IRQ_TYPE_NONE, - 0); + h->chip = chip; + return interrupt_add_handler(h); } +/* + * interrupt_remove_handler() - Remove a registered interrupt handler + * @hdl Interrupt handler to remove + * This function is the counterpart of interrupt_add_handler(). + * This function may panic on non-NULL invalid @hdl reference. + */ +void interrupt_remove_handler(struct itr_handler *hdl); + +/* + * interrupt_alloc_add_handler() - Allocate and register an interrupt handler + * @chip Interrupt controller + * @itr_num Interrupt number + * @handler Handler function + * @flags Bitmask flag ITRF_* + * @data Private data reference passed to @handler + * @out_hdl NULL or output pointer to allocated struct itr_handler + */ +TEE_Result interrupt_alloc_add_handler(struct itr_chip *chip, size_t it_num, + itr_handler_t handler, uint32_t flags, + void *data, + struct itr_handler **out_hdl); + +/* + * interrupt_remove_free_handler() - Remove/free a registered interrupt handler + * @hdl Interrupt handler to remove and free + * This function is the counterpart of interrupt_alloc_add_handler(). + * This function may panic on non-NULL invalid @hdl reference. + */ +void interrupt_remove_free_handler(struct itr_handler *hdl); + +/* + * itr_dt_get_func - Typedef of function to get an interrupt in DT node + * + * @args Reference to phandle arguments + * @data Pointer to data given at interrupt_register_provider() call + * @res Output result code of the operation: + * TEE_SUCCESS in case of success + * TEE_ERROR_DEFER_DRIVER_INIT if controller is not initialized + * Any TEE_Result compliant code in case of error. + * + * Returns an allocated struct itr_desc pointer referencing to an interrupt + * or NULL if invalid description in which case @res provides the error + * code. The interrupt matches the reference found with properties of the + * pointer FDT node. + * + * Upon success, the interrupt is configured and consumer can add a handler + * function to the interrupt. Yet, the interrupt is not enabled until consumer + * calls interrupt_enable(). + * Upon success, struct itr_desc must point to memory allocated with malloc() + * or like as it is freed prior returning to caller function. + */ +typedef struct itr_desc *(*itr_dt_get_func)(struct dt_pargs *args, void *data, + TEE_Result *res); + +#ifdef CFG_DT +/** + * interrupt_register_provider() - Register an interrupt provider + * + * @fdt Device tree to work on + * @node Node offset of the interrupt controller in the DT + * @dt_get_itr Callback to match the devicetree interrupt reference with + * @data Data which will be passed to the get_dt_its callback + */ +TEE_Result interrupt_register_provider(const void *fdt, int node, + itr_dt_get_func dt_get_itr, void *data); + +/** + * dt_get_interrupt_by_index() - Get an interrupt from DT by interrupt index + * + * Interrupt index (@index) refers to the index of the target interrupt to be + * retrieved as DT binding property "interrupts" may define several + * interrupts. + * + * @fdt Device tree to work on + * @node Node offset of the subnode containing interrupt(s) references + * @index Index in "interrupts" or "extended-interrupts" property list + * @chip Output interrupt controller reference upon success + * @itr_num Output interrupt number upon success + * + * Return TEE_SUCCESS in case of success + * Return TEE_ERROR_DEFER_DRIVER_INIT if interrupt controller is not yet inited + * Return TEE_ERROR_ITEM_NOT_FOUND if the DT does not reference target interrupt + * Return any other TEE_Result compliant code in case of error + */ +TEE_Result dt_get_interrupt_by_index(const void *fdt, int node, + unsigned int index, struct itr_chip **chip, + size_t *itr_num); + +/** + * dt_get_interrupt_by_name() - Get an interrupt from DT by interrupt name + * + * @fdt Device tree to work on + * @node Node offset of the subnode containing interrupt(s) references + * @name Name identifier used in "interrupt-names" property + * @chip Output interrupt controller reference upon success + * @itr_num Output interrupt number upon success + * + * Return TEE_SUCCESS in case of success + * Return TEE_ERROR_DEFER_DRIVER_INIT if interrupt controller is not yet inited + * Return TEE_ERROR_ITEM_NOT_FOUND if the DT does not reference target interrupt + * Return any other TEE_Result compliant code in case of error + */ +TEE_Result dt_get_interrupt_by_name(const void *fdt, int node, const char *name, + struct itr_chip **chip, size_t *itr_num); +#else +static inline TEE_Result interrupt_register_provider(const void *dt __unused, + int node __unused, + itr_dt_get_func f __unused, + void *data __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +static inline TEE_Result dt_get_interrupt_by_index(const void *fdt __unused, + int node __unused, + unsigned int index __unused, + struct itr_chip **c __unused, + size_t *itr_num __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} + +static inline TEE_Result dt_get_interrupt_by_name(const void *fdt __unused, + int node __unused, + const char *name __unused, + struct itr_chip **ch __unused, + size_t *itr_num __unused) +{ + return TEE_ERROR_NOT_IMPLEMENTED; +} +#endif /*CFG_DT*/ + +/* + * Helper function for when caller retrieves the first interrupt defined + * in "interrupts" or "extended-interrupts" DT binding property list. + */ +static inline TEE_Result dt_get_interrupt(const void *fdt, int node, + struct itr_chip **chip, + size_t *itr_num) +{ + return dt_get_interrupt_by_index(fdt, node, 0, chip, itr_num); +} #endif /*__KERNEL_INTERRUPT_H*/ diff --git a/core/kernel/dt_driver.c b/core/kernel/dt_driver.c index 3ab73f34eee..e53ad600d1e 100644 --- a/core/kernel/dt_driver.c +++ b/core/kernel/dt_driver.c @@ -109,6 +109,7 @@ static void assert_type_is_valid(enum dt_driver_type type) case DT_DRIVER_GPIO: case DT_DRIVER_I2C: case DT_DRIVER_PINCTRL: + case DT_DRIVER_INTERRUPT: return; default: assert(0); @@ -185,6 +186,9 @@ int fdt_get_dt_driver_cells(const void *fdt, int nodeoffset, case DT_DRIVER_CLK: cells_name = "#clock-cells"; break; + case DT_DRIVER_INTERRUPT: + cells_name = "#interrupt-cells"; + break; case DT_DRIVER_RSTCTRL: cells_name = "#reset-cells"; break; @@ -288,6 +292,47 @@ void *dt_driver_device_from_parent(const void *fdt, int nodeoffset, return device_from_provider_prop(prv, fdt, nodeoffset, NULL, res); } +void *dt_driver_device_from_node_idx_prop_phandle(const char *prop_name, + const void *fdt, int nodeoffs, + unsigned int prop_index, + enum dt_driver_type type, + uint32_t phandle, + TEE_Result *res) +{ + int len = 0; + const uint32_t *prop = NULL; + int phandle_node_unused = -1; + struct dt_driver_provider *prv = NULL; + + prop = fdt_getprop(fdt, nodeoffs, prop_name, &len); + if (!prop) { + if (len != -FDT_ERR_NOTFOUND) { + DMSG("Corrupted node %s", prop_name); + *res = TEE_ERROR_GENERIC; + } else { + DMSG("Property %s missing in node %s", prop_name, + fdt_get_name(fdt, nodeoffs, NULL)); + *res = TEE_ERROR_ITEM_NOT_FOUND; + } + return NULL; + } + + prv = dt_driver_get_provider_by_phandle(phandle, type); + if (!prv) { + *res = TEE_ERROR_DEFER_DRIVER_INIT; + return NULL; + } + + prop_index *= dt_driver_provider_cells(prv); + if (prop_index * sizeof(*prop) >= (size_t)len) { + *res = TEE_ERROR_ITEM_NOT_FOUND; + return NULL; + } + + return device_from_provider_prop(prv, fdt, phandle_node_unused, + prop + prop_index, res); +} + void *dt_driver_device_from_node_idx_prop(const char *prop_name, const void *fdt, int nodeoffset, unsigned int prop_idx, diff --git a/core/kernel/interrupt.c b/core/kernel/interrupt.c index 9a2443a953f..19c7ba928dc 100644 --- a/core/kernel/interrupt.c +++ b/core/kernel/interrupt.c @@ -19,13 +19,30 @@ * we begin to modify settings after boot initialization. */ -static struct itr_chip *itr_chip __nex_bss; -static SLIST_HEAD(, itr_handler) handlers __nex_data = - SLIST_HEAD_INITIALIZER(handlers); +static struct itr_chip *itr_main_chip __nex_bss; + +TEE_Result itr_chip_init(struct itr_chip *chip) +{ + if (!itr_chip_is_valid(chip)) + return TEE_ERROR_BAD_PARAMETERS; + + SLIST_INIT(&chip->handlers); + + return TEE_SUCCESS; +} void interrupt_main_init(struct itr_chip *chip) { - itr_chip = chip; + if (itr_chip_init(chip)) + panic(); + + itr_main_chip = chip; +} + +struct itr_chip *interrupt_get_main_chip(void) +{ + assert(itr_main_chip); + return itr_main_chip; } #ifdef CFG_DT @@ -36,24 +53,35 @@ int dt_get_irq_type_prio(const void *fdt, int node, uint32_t *type, int count = 0; int it_num = DT_INFO_INVALID_INTERRUPT; - if (!itr_chip || !itr_chip->dt_get_irq) + if (!itr_main_chip || !itr_main_chip->dt_get_irq) return it_num; prop = fdt_getprop(fdt, node, "interrupts", &count); if (!prop) return it_num; - return itr_chip->dt_get_irq(prop, count, type, prio); + return itr_main_chip->dt_get_irq(prop, count, type, prio); } #endif -void itr_handle(size_t it) +/* This function is supposed to be overridden in platform specific code */ +void __weak __noreturn interrupt_main_handler(void) +{ + panic("Secure interrupt handler not defined"); +} + +/* + * Interrupt controller chip support + */ +void interrupt_call_handlers(struct itr_chip *chip, size_t itr_num) { struct itr_handler *h = NULL; bool was_handled = false; - SLIST_FOREACH(h, &handlers, link) { - if (h->it == it) { + assert(chip); + + SLIST_FOREACH(h, &chip->handlers, link) { + if (h->it == itr_num) { if (h->handler(h) == ITRR_HANDLED) was_handled = true; else if (!(h->flags & ITRF_SHARED)) @@ -62,79 +90,223 @@ void itr_handle(size_t it) } if (!was_handled) { - EMSG("Disabling unhandled interrupt %zu", it); - itr_chip->ops->disable(itr_chip, it); + EMSG("Mask unhandled interrupt %s:%zu", chip->name, itr_num); + interrupt_mask(chip, itr_num); } } -struct itr_handler *itr_alloc_add_type_prio(size_t it, itr_handler_t handler, - uint32_t flags, void *data, - uint32_t type, uint32_t prio) +TEE_Result interrupt_configure(struct itr_chip *chip, size_t itr_num, + uint32_t type, uint32_t prio) { - struct itr_handler *hdl = calloc(1, sizeof(*hdl)); + chip->ops->add(chip, itr_num, type, prio); - if (hdl) { - hdl->it = it; - hdl->handler = handler; - hdl->flags = flags; - hdl->data = data; - itr_add_type_prio(hdl, type, prio); + return TEE_SUCCESS; +} + +TEE_Result interrupt_add_configure_handler(struct itr_handler *hdl, + uint32_t type, uint32_t prio) +{ + struct itr_handler *h = NULL; + + assert(hdl && hdl->chip->ops); + + SLIST_FOREACH(h, &hdl->chip->handlers, link) { + if (h->it == hdl->it && + (!(hdl->flags & ITRF_SHARED) || + !(h->flags & ITRF_SHARED))) { + EMSG("Shared and non-shared flags on interrupt %s#%zu", + hdl->chip->name, hdl->it); + return TEE_ERROR_GENERIC; + } } - return hdl; + interrupt_configure(hdl->chip, hdl->it, type, prio); + + SLIST_INSERT_HEAD(&hdl->chip->handlers, hdl, link); + + return TEE_SUCCESS; } -void itr_free(struct itr_handler *hdl) +void interrupt_remove_handler(struct itr_handler *hdl) { + struct itr_handler *h = NULL; + bool disable_itr = true; + if (!hdl) return; - itr_chip->ops->disable(itr_chip, hdl->it); + SLIST_FOREACH(h, &hdl->chip->handlers, link) + if (h == hdl) + break; + if (!h) { + DMSG("Invalid %s:%zu", hdl->chip->name, hdl->it); + assert(false); + return; + } + + if (hdl->flags & ITRF_SHARED) { + SLIST_FOREACH(h, &hdl->chip->handlers, link) { + if (h != hdl && h->it == hdl->it) { + disable_itr = false; + break; + } + } + } + + if (disable_itr) + interrupt_disable(hdl->chip, hdl->it); - SLIST_REMOVE(&handlers, hdl, itr_handler, link); - free(hdl); + SLIST_REMOVE(&hdl->chip->handlers, hdl, itr_handler, link); } -void itr_add_type_prio(struct itr_handler *h, uint32_t type, uint32_t prio) +TEE_Result interrupt_alloc_add_handler(struct itr_chip *chip, size_t itr_num, + itr_handler_t handler, uint32_t flags, + void *data, struct itr_handler **out_hdl) { - struct itr_handler __maybe_unused *hdl = NULL; + TEE_Result res = TEE_ERROR_GENERIC; + struct itr_handler *hdl = NULL; - SLIST_FOREACH(hdl, &handlers, link) - if (hdl->it == h->it) - assert((hdl->flags & ITRF_SHARED) && - (h->flags & ITRF_SHARED)); + hdl = calloc(1, sizeof(*hdl)); + if (!hdl) + return TEE_ERROR_OUT_OF_MEMORY; + + *hdl = ITR_HANDLER(chip, itr_num, flags, handler, data); + + res = interrupt_add_handler(hdl); + if (res) { + free(hdl); + return res; + } + + if (out_hdl) + *out_hdl = hdl; - itr_chip->ops->add(itr_chip, h->it, type, prio); - SLIST_INSERT_HEAD(&handlers, h, link); + return TEE_SUCCESS; } -void itr_enable(size_t it) +void interrupt_remove_free_handler(struct itr_handler *hdl) { - itr_chip->ops->enable(itr_chip, it); + if (hdl) { + interrupt_remove_handler(hdl); + free(hdl); + } } -void itr_disable(size_t it) +#ifdef CFG_DT +TEE_Result interrupt_register_provider(const void *fdt, int node, + itr_dt_get_func dt_get_itr, void *data) { - itr_chip->ops->disable(itr_chip, it); + return dt_driver_register_provider(fdt, node, + (get_of_device_func)dt_get_itr, + data, DT_DRIVER_INTERRUPT); } -void itr_raise_pi(size_t it) +/* + * Provide an itr_desc reference based on "interrupts" property bindings. + * May return TEE_ERROR_DEFER_DRIVER_INIT if parent controller is found but + * not yet initialized. + */ +static TEE_Result get_legacy_interrupt_by_index(const void *fdt, int node, + unsigned int index, + struct itr_desc **desc) { - itr_chip->ops->raise_pi(itr_chip, it); + TEE_Result res = TEE_ERROR_GENERIC; + const uint32_t *prop = NULL; + uint32_t phandle = 0; + int pnode = 0; + int len = 0; + + prop = fdt_getprop(fdt, node, "interrupts", &len); + if (!prop) + return TEE_ERROR_ITEM_NOT_FOUND; + + /* Find "interrupt-parent" in node or its parents */ + pnode = node; + prop = fdt_getprop(fdt, pnode, "interrupt-parent", &len); + + while (!prop) { + pnode = fdt_parent_offset(fdt, pnode); + if (pnode < 0) + break; + + prop = fdt_getprop(fdt, pnode, "interrupt-parent", &len); + if (!prop && len != -FDT_ERR_NOTFOUND) + break; + } + if (!prop) { + DMSG("No interrupt parent for node %s", + fdt_get_name(fdt, node, NULL)); + return TEE_ERROR_GENERIC; + } + + /* "interrupt-parent" provides interrupt controller phandle */ + phandle = fdt32_to_cpu(prop[0]); + + /* Get interrupt chip/number from phandle and "interrupts" property */ + *desc = dt_driver_device_from_node_idx_prop_phandle("interrupts", fdt, + node, index, + DT_DRIVER_INTERRUPT, + phandle, &res); + return res; } -void itr_raise_sgi(size_t it, uint8_t cpu_mask) +/* + * Provide an itr_desc based on "interrupts-extended" property bindings. + * May return TEE_ERROR_DEFER_DRIVER_INIT if parent controller is found + * but not yet initialized. + * With this function, provider is expected to have allocated itr_desc + * with malloc() or like. + */ +static TEE_Result get_extended_interrupt_by_index(const void *fdt, int node, + unsigned int index, + struct itr_desc **desc) { - itr_chip->ops->raise_sgi(itr_chip, it, cpu_mask); + TEE_Result res = TEE_ERROR_GENERIC; + + *desc = dt_driver_device_from_node_idx_prop("interrupts-extended", + fdt, node, index, + DT_DRIVER_INTERRUPT, &res); + + return res; } -void itr_set_affinity(size_t it, uint8_t cpu_mask) +TEE_Result dt_get_interrupt_by_index(const void *fdt, int node, + unsigned int index, struct itr_chip **chip, + size_t *itr_num) { - itr_chip->ops->set_affinity(itr_chip, it, cpu_mask); + TEE_Result res = TEE_ERROR_GENERIC; + struct itr_desc *desc = NULL; + + assert(chip && itr_num); + + /* "interrupts-extended" takes precedence over "interrupts" */ + if (fdt_getprop(fdt, node, "interrupts-extended", NULL)) + res = get_extended_interrupt_by_index(fdt, node, index, &desc); + else + res = get_legacy_interrupt_by_index(fdt, node, index, &desc); + + assert((!res && desc) || (res && !desc)); + + if (!res) { + *chip = desc->chip; + *itr_num = desc->itr_num; + + /* Balance malloc() or like from itr_dt_get_func callback */ + free(desc); + } + + return res; } -/* This function is supposed to be overridden in platform specific code */ -void __weak __noreturn interrupt_main_handler(void) +TEE_Result dt_get_interrupt_by_name(const void *fdt, int node, const char *name, + struct itr_chip **chip, size_t *itr_num) { - panic("Secure interrupt handler not defined"); + int idx = 0; + + idx = fdt_stringlist_search(fdt, node, "interrupt-names", name); + if (idx < 0) + return TEE_ERROR_GENERIC; + + return dt_get_interrupt_by_index(fdt, node, idx, chip, itr_num); } +#endif /*CFG_DT*/ diff --git a/core/kernel/notif.c b/core/kernel/notif.c index 7ecc246fdf5..c732818a646 100644 --- a/core/kernel/notif.c +++ b/core/kernel/notif.c @@ -93,6 +93,7 @@ uint32_t notif_get_value(bool *value_valid, bool *value_pending) void notif_send_async(uint32_t value) { uint32_t old_itr_status = 0; + struct itr_chip *itr_chip = interrupt_get_main_chip(); static_assert(CFG_CORE_ASYNC_NOTIF_GIC_INTID >= GIC_PPI_BASE); @@ -101,7 +102,7 @@ void notif_send_async(uint32_t value) DMSG("0x%"PRIx32, value); bit_set(notif_values, value); - itr_raise_pi(CFG_CORE_ASYNC_NOTIF_GIC_INTID); + itr_chip->ops->raise_pi(itr_chip, CFG_CORE_ASYNC_NOTIF_GIC_INTID); cpu_spin_unlock_xrestore(¬if_lock, old_itr_status); }