diff --git a/platform/loongarch64/ls3a5000/board.rs b/platform/loongarch64/ls3a5000/board.rs index 0b1f13905..82bc582e0 100644 --- a/platform/loongarch64/ls3a5000/board.rs +++ b/platform/loongarch64/ls3a5000/board.rs @@ -77,8 +77,11 @@ pub const ROOT_ZONE_MEMORY_REGIONS: &[HvConfigMemoryRegion] = &[ mem_type: MEM_TYPE_IO, physical_start: 0x1fe00000, virtual_start: 0x1fe00000, - size: 0x1000, - }, // uart0 + size: 0x3000, + }, // IOCSR MMIO: uart0, IPI, EXTIOI. + // Reference: + // IPI: + // EXTIOI: HvConfigMemoryRegion { mem_type: MEM_TYPE_IO, physical_start: 0x10080000, diff --git a/platform/loongarch64/ls3a6000/board.rs b/platform/loongarch64/ls3a6000/board.rs index fdc6f0185..f611b52b9 100644 --- a/platform/loongarch64/ls3a6000/board.rs +++ b/platform/loongarch64/ls3a6000/board.rs @@ -17,68 +17,127 @@ use crate::pci_dev; use crate::{arch::zone::HvArchZoneConfig, config::*, pci::vpci_dev::VpciDevType}; -pub const BOARD_NAME: &str = "ls3a5000"; +pub const BOARD_NAME: &str = "ls3a6000"; pub const BOARD_NCPUS: usize = 4; pub const ROOT_ZONE_DTB_ADDR: u64 = 0x10000f000; pub const ROOT_ZONE_KERNEL_ADDR: u64 = 0x200000; -pub const ROOT_ZONE_ENTRY: u64 = 0x9000000000d8c000; -pub const ROOT_ZONE_CPUS: u64 = 1 << 0; +pub const ROOT_ZONE_ENTRY: u64 = 0x9000000000dc6000; +pub const ROOT_ZONE_CPUS: u64 = (1 << 0) | (1 << 1); pub const ROOT_ZONE_NAME: &str = "root-linux-la64"; pub const ROOT_ZONE_MEMORY_REGIONS: &[HvConfigMemoryRegion] = &[ - /* memory regions */ + /* Legacy low RAM used during early boot. */ + HvConfigMemoryRegion { + mem_type: MEM_TYPE_RAM, + physical_start: 0x1000, + virtual_start: 0x0, + size: 0x10000, + }, + HvConfigMemoryRegion { + mem_type: MEM_TYPE_RAM, + physical_start: 0x10000, + virtual_start: 0x10000, + size: 0x1f0000, + }, + /* Keep these banks aligned with the root Linux device tree. */ HvConfigMemoryRegion { mem_type: MEM_TYPE_RAM, physical_start: 0x00200000, virtual_start: 0x00200000, - size: 0x0ee00000, - }, // ram + size: 0x0ec00000, + }, + HvConfigMemoryRegion { + mem_type: MEM_TYPE_RAM, + physical_start: 0x90400000, + virtual_start: 0x90400000, + size: 0x67b60000, + }, + HvConfigMemoryRegion { + mem_type: MEM_TYPE_RAM, + physical_start: 0xf7f70000, + virtual_start: 0xf7f70000, + size: 0x05f10000, + }, + HvConfigMemoryRegion { + mem_type: MEM_TYPE_RAM, + physical_start: 0xfe440000, + virtual_start: 0xfe440000, + size: 0xf1bc0000, + }, + /* Shared-memory and non-root banks reserved by the root device tree. */ + HvConfigMemoryRegion { + mem_type: MEM_TYPE_RAM, + physical_start: 0x2_00c0_0000, + virtual_start: 0x2_00c0_0000, + size: 0x0400_0000, + }, + HvConfigMemoryRegion { + mem_type: MEM_TYPE_RAM, + physical_start: 0x2_04c0_0000, + virtual_start: 0x2_04c0_0000, + size: 0x0400_0000, + }, + HvConfigMemoryRegion { + mem_type: MEM_TYPE_RAM, + physical_start: 0x2_08c0_0000, + virtual_start: 0x2_08c0_0000, + size: 0x0400_0000, + }, + HvConfigMemoryRegion { + mem_type: MEM_TYPE_RAM, + physical_start: 0x2_0cc0_0000, + virtual_start: 0x2_0cc0_0000, + size: 0x0400_0000, + }, HvConfigMemoryRegion { mem_type: MEM_TYPE_RAM, - physical_start: 0x90000000, - virtual_start: 0x90000000, - size: 0x10000000, - }, // ram + physical_start: 0x2_1cc0_0000, + virtual_start: 0x2_1cc0_0000, + size: 0x1_0000_0000, + }, HvConfigMemoryRegion { mem_type: MEM_TYPE_RAM, - physical_start: 0xf000_0000, - virtual_start: 0xf000_0000, - size: 0x1000_0000, - }, // ram + physical_start: 0x3_1cc0_0000, + virtual_start: 0x3_1cc0_0000, + size: 0x1_0000_0000, + }, HvConfigMemoryRegion { mem_type: MEM_TYPE_RAM, - physical_start: 0x1_6000_0000, - virtual_start: 0x1_6000_0000, - size: 0x1000_0000, - }, // linux0 + physical_start: 0x4_1cc0_0000, + virtual_start: 0x4_1cc0_0000, + size: 0x1_0000_0000, + }, HvConfigMemoryRegion { mem_type: MEM_TYPE_RAM, - physical_start: 0xc000_0000, - virtual_start: 0xc000_0000, - size: 0x3000_0000, - }, // linux1 + physical_start: 0x5_1cc0_0000, + virtual_start: 0x5_1cc0_0000, + size: 0x1_0000_0000, + }, HvConfigMemoryRegion { mem_type: MEM_TYPE_RAM, - physical_start: 0xa000_0000, - virtual_start: 0xa000_0000, - size: 0x2000_0000, - }, // linux2 + physical_start: 0x6_1cc0_0000, + virtual_start: 0x6_1cc0_0000, + size: 0x1_0000_0000, + }, HvConfigMemoryRegion { mem_type: MEM_TYPE_RAM, - physical_start: 0x1_0000_0000, - virtual_start: 0x1_0000_0000, - size: 0x2000_0000, - }, // linux3 + physical_start: 0x7_1cc0_0000, + virtual_start: 0x7_1cc0_0000, + size: 0x1_0000_0000, + }, /* devices and controllers */ HvConfigMemoryRegion { mem_type: MEM_TYPE_IO, physical_start: 0x1fe00000, virtual_start: 0x1fe00000, - size: 0x1000, - }, // uart0 + size: 0x3000, + }, // IOCSR MMIO: uart0, IPI, EXTIOI. + // Reference: + // IPI: + // EXTIOI: HvConfigMemoryRegion { mem_type: MEM_TYPE_IO, physical_start: 0x10080000, @@ -97,6 +156,18 @@ pub const ROOT_ZONE_MEMORY_REGIONS: &[HvConfigMemoryRegion] = &[ virtual_start: 0x10000000, size: 0x1000, }, // pch-pic irq controller + HvConfigMemoryRegion { + mem_type: MEM_TYPE_IO, + physical_start: 0x100A_0000, + virtual_start: 0x100A_0000, + size: 0x1000, + }, // LS7A PWM0-3 + HvConfigMemoryRegion { + mem_type: MEM_TYPE_IO, + physical_start: 0x0E0010000000, + virtual_start: 0x0E0010000000, + size: 0x1000, + }, // ACPI thermal sensor /* PCI related stuffs ... */ // HvConfigMemoryRegion { // mem_type: MEM_TYPE_IO, @@ -115,40 +186,25 @@ pub const ROOT_ZONE_MEMORY_REGIONS: &[HvConfigMemoryRegion] = &[ // physical_start: 0x18408000, // virtual_start: 0x18408000, // size: 0x00008000, - // }, // pci io resource - // HvConfigMemoryRegion { - // mem_type: MEM_TYPE_IO, - // physical_start: 0x60000000, - // virtual_start: 0x60000000, - // size: 0x20000000, - // }, // pci mem resource + // }, // pci config space (HT) HvConfigMemoryRegion { mem_type: MEM_TYPE_IO, physical_start: 0x1001_0000, virtual_start: 0x1001_0000, size: 0x0001_0000, - }, // ? - /* map special regions - 2024.4.12 */ - // linux's strscpy called gpa at 0x9000_0000_0000_0000 which is ldx x, 0x9000_0000_0000_0000(a1) + 0x0(a0) why ? - // __memcpy_fromio 0xf0000 why? + }, // PCH/LS7A miscellaneous registers HvConfigMemoryRegion { - mem_type: MEM_TYPE_RAM, - physical_start: 0x1000, - virtual_start: 0x0, - size: 0x10000, - }, // 0x0 - HvConfigMemoryRegion { - mem_type: MEM_TYPE_RAM, - physical_start: 0xf0000, - virtual_start: 0xf0000, - size: 0x10000, - }, // 0xf0000 + mem_type: MEM_TYPE_IO, + physical_start: 0x18408000, + virtual_start: 0x18408000, + size: 0x00008000, + }, // PCI IO resource HvConfigMemoryRegion { - mem_type: MEM_TYPE_RAM, - physical_start: 0x1_4000_0000, - virtual_start: 0x1_4000_0000, - size: 0x80_0000, // linux1-root - }, // SHARD_MEM + mem_type: MEM_TYPE_IO, + physical_start: 0x60000000, + virtual_start: 0x60000000, + size: 0x20000000, + }, // PCI memory resource ]; pub const IRQ_WAKEUP_VIRTIO_DEVICE: usize = 32 + 0x20; @@ -158,7 +214,7 @@ pub const ROOT_ZONE_IVC_CONFIG: [HvIvcConfig; 0] = []; pub const ROOT_PCI_CONFIG: [HvPciConfig; 1] = [HvPciConfig { bus_range_begin: 0x0, - bus_range_end: 0x1f, + bus_range_end: 0xff, ecam_base: 0xfe00000000, ecam_size: 0x20000000, io_base: 0x18408000, diff --git a/platform/loongarch64/ls3a6000/configs/zone1-linux.json b/platform/loongarch64/ls3a6000/configs/zone1-linux.json index cbfbf9728..08f21c7fb 100644 --- a/platform/loongarch64/ls3a6000/configs/zone1-linux.json +++ b/platform/loongarch64/ls3a6000/configs/zone1-linux.json @@ -3,32 +3,15 @@ "name": "linux1", "zone_id": 1, "cpus": [ - 1 + 2, + 3 ], "memory_regions": [ { "type": "ram", - "physical_start": "0xc0000000", - "virtual_start": "0xc0000000", - "size": "0x10000000" - }, - { - "type": "ram", - "physical_start": "0xd0000000", - "virtual_start": "0xd0000000", - "size": "0x10000000" - }, - { - "type": "io", - "physical_start": "0x1fe00000", - "virtual_start": "0x1fe00000", - "size": "0x1000" - }, - { - "type": "io", - "physical_start": "0x10080000", - "virtual_start": "0x10080000", - "size": "0x1000" + "physical_start": "0x21cc00000", + "virtual_start": "0x21cc00000", + "size": "0x200000000" }, { "type": "virtio", @@ -42,30 +25,6 @@ "virtual_start": "0x30002000", "size": "0x200" }, - { - "type": "io", - "physical_start": "0xffffffff0000", - "virtual_start": "0xffffffff0000", - "size": "0x1000" - }, - { - "type": "io", - "physical_start": "0x10000000", - "virtual_start": "0x10000000", - "size": "0x1000" - }, - { - "type": "io", - "physical_start": "0x100d0000", - "virtual_start": "0x100d0000", - "size": "0x1000" - }, - { - "type": "io", - "physical_start": "0x10010000", - "virtual_start": "0x10010000", - "size": "0x00010000" - }, { "type": "ram", "physical_start": "0x1000", @@ -80,34 +39,49 @@ }, { "type": "ram", - "physical_start": "0x140000000", - "virtual_start": "0x140000000", - "size": "0x800000" + "physical_start": "0x200c00000", + "virtual_start": "0x200c00000", + "size": "0x04000000" }, { "type": "ram", - "physical_start": "0x140800000", - "virtual_start": "0x140800000", - "size": "0x800000" + "physical_start": "0x204c00000", + "virtual_start": "0x204c00000", + "size": "0x04000000" }, { "type": "ram", - "physical_start": "0x141000000", - "virtual_start": "0x141000000", - "size": "0x800000" + "physical_start": "0x210c00000", + "virtual_start": "0x210c00000", + "size": "0x04000000" + }, + { + "type": "ram", + "physical_start": "0x214c00000", + "virtual_start": "0x214c00000", + "size": "0x04000000" } ], "interrupts": [ 4, 5 ], - "ivc_configs": [], + "ivc_configs": [{ + "ivc_id": 0, + "peer_id": 1, + "control_table_ipa": "0x200be0000", + "shared_mem_ipa": "0x200be1000", + "rw_sec_size": 0, + "out_sec_size": "0x1000", + "interrupt_num": 148, + "max_peers": 2 + }], "kernel_args": "NOT_USED_YET", "kernel_filepath": "/tool/nonroot/vmlinux-linux1.bin", "dtb_filepath": "/tool/test.bin", - "kernel_load_paddr": "0xc0200000", - "dtb_load_paddr": "0xc0000000", - "entry_point": "0x90000000c0dff000", + "kernel_load_paddr": "0x21ce00000", + "dtb_load_paddr": "0x21cc00000", + "entry_point": "0x900000021da22000", "arch_config": { "dummy": "0x1234" }, @@ -120,19 +94,22 @@ "mem32_base": "0x0", "mem32_size": "0x0", "pci_mem32_base": "0x0", - "mem64_base": "0x60000000", - "mem64_size": "0x20000000", - "pci_mem64_base": "0x60000000", - "bus_range_begin": "0x0", - "bus_range_end": "0x1f", + "mem64_base": "0x75300000", + "mem64_size": "0x1000000", + "pci_mem64_base": "0x75300000", + "bus_range_begin": "0x6", + "bus_range_end": "0x6", "domain": "0x0" }], "num_pci_devs": 1, "alloc_pci_devs": [{ "domain": "0x0", "bus": "0x6", - "device": "0x1", - "function": "0x0", + "device": "0x0", + "function": "0x1", + "v_bus": "0x6", + "v_device": "0x0", + "v_function": "0x1", "dev_type": "0" }] -} \ No newline at end of file +} diff --git a/platform/loongarch64/ls3a6000/image/dts/include/loongson-3a5000.dtsi b/platform/loongarch64/ls3a6000/image/dts/include/loongson-3a5000.dtsi index 3880aca0a..05fef3973 100644 --- a/platform/loongarch64/ls3a6000/image/dts/include/loongson-3a5000.dtsi +++ b/platform/loongarch64/ls3a6000/image/dts/include/loongson-3a5000.dtsi @@ -21,6 +21,14 @@ next-level-cache = <&scache0>; numa-node-id = <0>; }; + cpu@1 { + compatible = "loongson,loongson3"; + device_type = "cpu"; + reg = <0x1>; + l2-cache = <&vcache1>; + next-level-cache = <&scache0>; + numa-node-id = <0>; + }; vcache0: l2-cache0 { compatible = "cache"; next-level-cache = <&scache0>; diff --git a/src/arch/loongarch64/cpu.rs b/src/arch/loongarch64/cpu.rs index c458efa75..4596bbfb4 100644 --- a/src/arch/loongarch64/cpu.rs +++ b/src/arch/loongarch64/cpu.rs @@ -73,15 +73,22 @@ impl ArchCpu { for i in 0..32 { self.ctx.x[i] = 0; } - // set all zone's GCSR.CPUID to 0 beacuse linux running on it will believe it's CPU0 - // - wheatfox 2025.5.20 - self.ctx.gcsr_cpuid = 0; + let zone_cpu_num = this_cpu_data() + .zone + .as_ref() + .map(|zone| zone.read().cpu_num()) + .unwrap_or(1); + self.ctx.gcsr_cpuid = if zone_cpu_num == 1 { + 0 + } else { + this_cpu_data().id + }; info!( "[[CPU virtualization]] CPU{} run@{:#x}", self.get_cpuid(), self.ctx.sepc ); - info!("loongarch64: @{:#x?}", self); + debug!("loongarch64: @{:#x?}", self); // step 1: enable guest mode // step 2: set guest entry to era // step 3: run ertn and enter guest mode diff --git a/src/arch/loongarch64/hypercall.rs b/src/arch/loongarch64/hypercall.rs index 6b56282f4..adf49e555 100644 --- a/src/arch/loongarch64/hypercall.rs +++ b/src/arch/loongarch64/hypercall.rs @@ -17,7 +17,6 @@ use crate::arch::cpu::this_cpu_id; use crate::config::HvZoneConfig; use crate::config::CONFIG_MAGIC_VERSION; -use crate::device::virtio_trampoline::MAX_DEVS; use crate::hypercall::HyperCall; use crate::hypercall::HyperCallResult; impl<'a> HyperCall<'a> { @@ -26,13 +25,6 @@ impl<'a> HyperCall<'a> { HyperCallResult::Ok(0) } - pub fn wait_for_interrupt(&mut self, irq_list: &mut [u64; MAX_DEVS + 1]) { - use crate::device::irqchip::ls7a2000::*; - let status = GLOBAL_IRQ_INJECT_STATUS.lock(); - drop(status); - irq_list[0] = 0; // CAUTION: this is a workaround for loongarch64 - } - pub fn hv_zone_config_check(&self, magic_version: *mut u64) -> HyperCallResult { let magic_version_raw = magic_version as u64; let magic_version_hva = magic_version_raw | crate::arch::mm::LOONGARCH64_CACHED_DMW_PREFIX; @@ -60,7 +52,7 @@ impl<'a> HyperCall<'a> { pub fn check_cpu_id(&self) { let cpuid = this_cpu_id(); - assert_eq!(cpuid, 0); + trace!("CPU ID: {} Start Zone", cpuid); } pub fn hv_virtio_get_irq(&self, virtio_irq: *mut u32) -> HyperCallResult { diff --git a/src/arch/loongarch64/ipi.rs b/src/arch/loongarch64/ipi.rs index 1e1f0cfdb..03741821e 100644 --- a/src/arch/loongarch64/ipi.rs +++ b/src/arch/loongarch64/ipi.rs @@ -15,7 +15,7 @@ // Yulong Han // use crate::arch::cpu::this_cpu_id; -use crate::consts::IPI_EVENT_CLEAR_INJECT_IRQ; +use crate::consts::{IPI_EVENT_CLEAR_INJECT_IRQ, IPI_EVENT_SEND_IPI}; use crate::device::common::MMIODerefWrapper; use core::arch::asm; use core::ptr::write_volatile; @@ -73,6 +73,25 @@ register_structs! { const MMIO_BASE: usize = 0x8000_0000_1fe0_0000; const IPI_MMIO_BASE: usize = MMIO_BASE; const IPI_ANY_SEND_BASE: usize = MMIO_BASE + 0x1158; +const IOCSR_IPI_STATUS: usize = 0x1000; +const IOCSR_IPI_ENABLE: usize = 0x1004; +const IOCSR_IPI_CLEAR: usize = 0x100c; + +#[inline] +fn iocsr_read32(reg: usize) -> u32 { + let value: usize; + unsafe { + asm!("iocsrrd.w {}, {}", out(reg) value, in(reg) reg); + } + value as u32 +} + +#[inline] +fn iocsr_write32(value: u32, reg: usize) { + unsafe { + asm!("iocsrwr.w {}, {}", in(reg) value as usize, in(reg) reg); + } +} // IPI registers, use this if you don't want to use the percore-IPI feature pub static CORE0_IPI: MMIODerefWrapper = @@ -89,7 +108,9 @@ pub const SMP_BOOT_CPU: usize = 0x1; pub const SMP_RESCHEDULE: usize = 0x2; pub const SMP_CALL_FUNCTION: usize = 0x4; // customized actions :), since there is no docs on this yet -pub const HVISOR_START_VCPU: usize = 0x8; +/// Dedicated physical IPI bit used only as the hvisor event-queue doorbell. +/// Linux SMP actions use bits 0..=2, so sharing those bits can drop guest IPI work. +pub const HVISOR_EVENT_DOORBELL: usize = 0x8; fn iocsr_mbuf_send_box_lo(a: usize) -> usize { a << 1 @@ -240,39 +261,26 @@ pub fn mail_send(data: usize, cpu_id: usize, mailbox_id: usize) { } pub fn enable_ipi(cpu_id: usize) { - let ipi: &MMIODerefWrapper = match cpu_id { - 0 => &CORE0_IPI, - 1 => &CORE1_IPI, - 2 => &CORE2_IPI, - 3 => &CORE3_IPI, - _ => { - error!("enable_ipi: invalid cpu_id: {}", cpu_id); - return; - } - }; - ipi.ipi_enable.write(IpiEnable::IPIENABLE.val(0xffffffff)); + assert_eq!(cpu_id, this_cpu_id()); + iocsr_write32(u32::MAX, IOCSR_IPI_ENABLE); debug!("enable_ipi: IPI enabled for cpu {}", cpu_id); } pub fn clear_all_ipi(cpu_id: usize) { - let ipi: &MMIODerefWrapper = match cpu_id { - 0 => &CORE0_IPI, - 1 => &CORE1_IPI, - 2 => &CORE2_IPI, - 3 => &CORE3_IPI, - _ => { - error!("clear_all_ipi: invalid cpu_id: {}", cpu_id); - return; - } - }; - ipi.ipi_clear.write(IpiClear::IPICLEAR.val(0xffffffff)); + assert_eq!(cpu_id, this_cpu_id()); + iocsr_write32(u32::MAX, IOCSR_IPI_CLEAR); debug!( "clear_all_ipi: IPI status for cpu {}: {:#x}", cpu_id, - ipi.ipi_status.read(IpiStatus::IPISTATUS) + iocsr_read32(IOCSR_IPI_STATUS) ); } +pub fn clear_ipi_bits(cpu_id: usize, mask: u32) { + assert_eq!(cpu_id, this_cpu_id()); + iocsr_write32(mask, IOCSR_IPI_CLEAR); +} + pub fn reset_ipi(cpu_id: usize) { // clear all IPIs and enable all IPIs clear_all_ipi(cpu_id); @@ -280,17 +288,8 @@ pub fn reset_ipi(cpu_id: usize) { } pub fn get_ipi_status(cpu_id: usize) -> u32 { - let ipi: &MMIODerefWrapper = match cpu_id { - 0 => &CORE0_IPI, - 1 => &CORE1_IPI, - 2 => &CORE2_IPI, - 3 => &CORE3_IPI, - _ => { - error!("get_ipi_status: invalid cpu_id: {}", cpu_id); - return 0; - } - }; - ipi.ipi_status.read(IpiStatus::IPISTATUS) + assert_eq!(cpu_id, this_cpu_id()); + iocsr_read32(IOCSR_IPI_STATUS) } pub fn ecfg_ipi_enable() { @@ -347,9 +346,10 @@ pub fn dump_ipi_registers() { pub fn arch_check_events(event: Option) { match event { Some(IPI_EVENT_CLEAR_INJECT_IRQ) => { - // clear the injected IPI interrupt - use crate::device::irqchip::ls7a2000::clear_hwi_injected_irq; - clear_hwi_injected_irq(); + warn!("legacy CLEAR_INJECT_IRQ event ignored; use the per-IRQ line API"); + } + Some(IPI_EVENT_SEND_IPI) => { + crate::arch::zone::sync_virtual_ipi_line(); } _ => { panic!("arch_check_events: unhandled event: {:?}", event); @@ -358,8 +358,6 @@ pub fn arch_check_events(event: Option) { } pub fn arch_prepare_send_event(cpu_id: usize, ipi_int_id: usize, event_id: usize) { - use crate::event::fetch_event; - while !fetch_event(cpu_id).is_none() {} debug!( "loongarch64:: send_event: cpu_id: {}, ipi_int_id: {}, event_id: {}", cpu_id, ipi_int_id, event_id diff --git a/src/arch/loongarch64/register/gintc.rs b/src/arch/loongarch64/register/gintc.rs index 26e3ece2c..3cf968d49 100644 --- a/src/arch/loongarch64/register/gintc.rs +++ b/src/arch/loongarch64/register/gintc.rs @@ -20,25 +20,49 @@ impl_define_csr!(Gintc, "GINTC"); impl_read_csr!(0x52, Gintc); impl Gintc { - pub fn hwis(&self) -> usize { + pub fn vip(&self) -> usize { self.bits.get_bits(0..=7) } - pub fn hwip(&self) -> usize { + pub fn pip(&self) -> usize { self.bits.get_bits(8..=15) } - pub fn hwic(&self) -> usize { + pub fn hc(&self) -> usize { self.bits.get_bits(16..=23) } + + pub fn hwis(&self) -> usize { + self.vip() + } + + pub fn hwip(&self) -> usize { + self.pip() + } + + pub fn hwic(&self) -> usize { + self.hc() + } +} + +pub fn write_vip(vip: usize) { + set_csr_loong_bits!(0x52, 0..=7, vip); +} + +pub fn write_pip(pip: usize) { + set_csr_loong_bits!(0x52, 8..=15, pip); +} + +pub fn write_hc(hc: usize) { + set_csr_loong_bits!(0x52, 16..=23, hc); } pub fn set_hwis(hwis: usize) { - set_csr_loong_bits!(0x52, 0..=7, hwis); + write_vip(hwis); } pub fn set_hwip(hwip: usize) { - set_csr_loong_bits!(0x52, 8..=15, hwip); + write_pip(hwip); } pub fn set_hwic(hwic: usize) { - set_csr_loong_bits!(0x52, 16..=23, hwic); + write_hc(hwic); } diff --git a/src/arch/loongarch64/trap.rs b/src/arch/loongarch64/trap.rs index e6f8b25b8..71b34183e 100644 --- a/src/arch/loongarch64/trap.rs +++ b/src/arch/loongarch64/trap.rs @@ -19,11 +19,11 @@ use super::register::*; use super::zone::ZoneContext; use crate::arch::cpu::this_cpu_id; use crate::arch::ipi::*; -use crate::consts::{IPI_EVENT_CLEAR_INJECT_IRQ, MAX_CPU_NUM}; +use crate::consts::MAX_CPU_NUM; use crate::cpu_data::this_cpu_data; use crate::device::irqchip::inject_irq; use crate::device::irqchip::ls7a2000::chip::*; -use crate::event::{check_events, dump_cpu_events, dump_events}; +use crate::event::{dump_events, handle_next_loongarch_event}; use crate::hypercall::{SGI_IPI_ID, *}; use crate::memory::{addr, mmio_handle_access, MMIOAccess}; use crate::zone::Zone; @@ -1275,28 +1275,28 @@ fn handle_interrupt(is: usize) { cpu_id, ipi_status ); - match ipi_status { - status if status == SGI_IPI_ID as _ => { - let events = dump_cpu_events(cpu_id); - debug!("CPU {} events: {:?}", cpu_id, events); - while check_events() {} - } - status if status == 0x8 => { - debug!("CPU {} received unhandled IPI status {:#x}", cpu_id, status); - } - status => { - warn!("CPU {} received unknown IPI status {:#x}", cpu_id, status); + let hvisor_mask = SGI_IPI_ID as u32; + if ipi_status & hvisor_mask != 0 { + clear_ipi_bits(cpu_id, hvisor_mask); + while handle_next_loongarch_event(SGI_IPI_ID as usize) { + clear_ipi_bits(cpu_id, hvisor_mask); } } - reset_ipi(cpu_id); + + let unhandled = ipi_status & !hvisor_mask; + if unhandled != 0 { + error!( + "CPU {} has unhandled physical IPI status {:#x}; preserving those bits", + cpu_id, unhandled + ); + } return; } // Handle timer interrupts if is & TIMER_BIT != 0 { - warn!("Timer interrupt received"); + debug!("Timer interrupt received"); loongArch64::register::ticlr::clear_timer_interrupt(); - crate::device::irqchip::ls7a2000::clear_hwi_injected_irq(); return; } @@ -1352,7 +1352,7 @@ fn emulate_cpucfg(ins: usize, ctx: &mut ZoneContext) { const MAX_CPUCFG_REGS: usize = 21; - info!( + debug!( "cpucfg emulation, target cpucfg index is {:#x}", cpucfg_target_idx ); @@ -1389,19 +1389,19 @@ fn emulate_csrx(ins: usize, ctx: &mut ZoneContext) { match ty { 0 => { // csrrd - info!("csrrd emulation for CSR {:#x}", csr); + debug!("csrrd emulation for CSR {:#x}", csr); ctx.x[rd] = 0; // just set it to 0 } 1 => { // csrwr - info!("csrwr emulation for CSR {:#x}", csr); + debug!("csrwr emulation for CSR {:#x}", csr); ctx.x[rd] = 0; // do nothing to GCSR, but we also need to set rd to 0 } _ => { // csrxchg - info!("csrxchg emulation for CSR {:#x}", csr); + debug!("csrxchg emulation for CSR {:#x}", csr); ctx.x[rd] = 0; // do nothing to GCSR, but we also need to set rd to 0 } diff --git a/src/arch/loongarch64/zone.rs b/src/arch/loongarch64/zone.rs index baa50d137..f1dbec423 100644 --- a/src/arch/loongarch64/zone.rs +++ b/src/arch/loongarch64/zone.rs @@ -16,24 +16,30 @@ // use crate::device::irqchip::ls7a2000::chip::get_extioi_sr; use crate::{ - arch::{cpu::this_cpu_id, trap::GLOBAL_TRAP_CONTEXT_HELPER_PER_CPU, Stage2PageTable}, + arch::{ + cpu::this_cpu_id, ipi::SMP_BOOT_CPU, trap::GLOBAL_TRAP_CONTEXT_HELPER_PER_CPU, + Stage2PageTable, + }, config::*, - consts::PAGE_SIZE, + consts::{IPI_EVENT_SEND_IPI, MAX_CPU_NUM, PAGE_SIZE}, + cpu_data::{get_cpu_data, this_cpu_data, VcpuState}, device::virtio_trampoline::mmio_virtio_handler, error::{HvError, HvResult}, + event::{send_event, IPI_EVENT_WAKEUP}, + hypercall::SGI_IPI_ID, memory::{ addr::{align_down, align_up}, mmio_generic_handler, mmio_perform_access, GuestPhysAddr, HostPhysAddr, MMIOAccess, MemFlags, MemoryRegion, MemorySet, }, - zone::Zone, + zone::{is_this_root_zone, Zone}, PHY_TO_DMW_UNCACHED, }; use alloc::boxed::Box; use alloc::collections::BTreeMap; use alloc::string::String; use alloc::vec::Vec; -use core::sync::atomic::{fence, AtomicU64, Ordering}; +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; use core::{arch::asm, ptr::write_volatile}; use spin::lazy::Lazy; use spin::Mutex; @@ -457,6 +463,12 @@ const LOG_INTERVAL: u64 = 100000; const BASE_ADDR: usize = PHY_TO_DMW_UNCACHED!(0x1fe0_0000); const UART0_BASE: usize = PHY_TO_DMW_UNCACHED!(0x1fe0_01e0); const UART0_SIZE: usize = 0x8; +const IPI_REG_BASE: usize = PHY_TO_DMW_UNCACHED!(0x1fe0_1000); +const IPI_REG_SIZE: usize = 0x400; +const IPI_SEND_BASE: usize = PHY_TO_DMW_UNCACHED!(0x1fe0_1040); +const IPI_SEND_SIZE: usize = 0x4; +const IPI_MAIL_SEND_BASE: usize = PHY_TO_DMW_UNCACHED!(0x1fe0_1048); +const IPI_MAIL_SEND_SIZE: usize = 0x8; const LIOINTC_MAP_BASE: usize = PHY_TO_DMW_UNCACHED!(0x1fe0_1400); // 1400-141f const LIOINTC_MAP_SIZE: usize = 0x20; const ANYSEND_BASE: usize = PHY_TO_DMW_UNCACHED!(0x1fe0_1158); @@ -484,6 +496,41 @@ macro_rules! is_in_mmio_range { }; } +const IPI_STATUS: usize = 0x00; +const IPI_ENABLE: usize = 0x04; +const IPI_SET: usize = 0x08; +const IPI_CLEAR: usize = 0x0c; +const IPI_MAILBOX_BASE: usize = 0x20; +const IPI_MAILBOX_SIZE: usize = 0x20; + +static VIRTUAL_IPI_STATUS: [AtomicU32; MAX_CPU_NUM] = { + const C: AtomicU32 = AtomicU32::new(0); + [C; MAX_CPU_NUM] +}; +static VIRTUAL_IPI_ENABLE: [AtomicU32; MAX_CPU_NUM] = { + const C: AtomicU32 = AtomicU32::new(0xffff_ffff); + [C; MAX_CPU_NUM] +}; +static VIRTUAL_IPI_MAILBOX: [AtomicU64; MAX_CPU_NUM * 4] = { + const C: AtomicU64 = AtomicU64::new(0); + [C; MAX_CPU_NUM * 4] +}; +static VIRTUAL_IPI_TRACE_SEQ: AtomicU64 = AtomicU64::new(0); +const VIRTUAL_IPI_TRACE_LIMIT: u64 = 64; +const IOCSR_IPI_SEND_BLOCKING: u32 = 1 << 31; + +pub fn reset_virtual_ipi_state(cpu: usize) { + if cpu >= MAX_CPU_NUM { + return; + } + + VIRTUAL_IPI_STATUS[cpu].store(0, Ordering::Release); + VIRTUAL_IPI_ENABLE[cpu].store(0xffff_ffff, Ordering::Release); + for mailbox in 0..4 { + VIRTUAL_IPI_MAILBOX[cpu * 4 + mailbox].store(0, Ordering::Release); + } +} + fn handle_uart_mmio(mmio: &mut MMIOAccess, base_addr: usize) -> HvResult { mmio_perform_access(base_addr, mmio); Ok(()) @@ -513,9 +560,9 @@ fn handle_extioi_mapping_mmio(mmio: &mut MMIOAccess, base_addr: usize, size: usi return Ok(()); } - // if this is nonroot, we ignore the mmio - if this_cpu_id() != 0 { - info!("nonroot's write to extioi mapping regs, ignored"); + // Non-root zones see a virtual CPU0 even when running on another physical CPU. + if !is_this_root_zone() { + debug!("nonroot's write to extioi mapping regs, ignored"); return Ok(()); } @@ -587,6 +634,279 @@ fn handle_generic_mmio(mmio: &mut MMIOAccess, base_addr: usize) -> HvResult { Ok(()) } +fn this_guest_cpu_id() -> usize { + this_cpu_data() + .zone + .as_ref() + .and_then(|zone| zone.read().phys_to_guest_cpu(this_cpu_id())) + .unwrap_or(0) +} + +fn guest_cpu_to_physical_cpu(guest_cpu: usize) -> Option { + this_cpu_data() + .zone + .as_ref() + .and_then(|zone| zone.read().guest_to_phys_cpu(guest_cpu)) +} + +pub fn sync_virtual_ipi_line() { + let cpu = this_cpu_id(); + let status = VIRTUAL_IPI_STATUS[cpu].load(Ordering::Acquire); + + if status != 0 { + crate::device::irqchip::ls7a2000::inject_irq(12, false); + } else { + crate::device::irqchip::ls7a2000::clear_injected_irq(12); + } +} + +fn sync_or_notify_virtual_ipi_line(cpu: usize) { + if cpu == this_cpu_id() { + sync_virtual_ipi_line(); + } else { + send_event(cpu, SGI_IPI_ID as usize, IPI_EVENT_SEND_IPI); + } +} + +fn trace_virtual_ipi_transition( + operation: &str, + target_cpu: usize, + bits: u32, + old: u32, + new: u32, + blocking: bool, +) { + let seq = VIRTUAL_IPI_TRACE_SEQ.fetch_add(1, Ordering::Relaxed); + if seq < VIRTUAL_IPI_TRACE_LIMIT { + info!( + "[HVDBG:guest-ipi] seq={} op={} src_cpu={} dst_cpu={} bits={:#x} old={:#x} new={:#x} blocking={}", + seq, + operation, + this_cpu_id(), + target_cpu, + bits, + old, + new, + blocking + ); + } +} + +fn virtual_ipi_send(target_guest_cpu: usize, ipi_bits: u32, blocking: bool) -> HvResult { + let Some(target_cpu) = guest_cpu_to_physical_cpu(target_guest_cpu) else { + warn!( + "loongarch64: guest IPI target vCPU {} out of range", + target_guest_cpu + ); + return Ok(()); + }; + + if ipi_bits & SMP_BOOT_CPU as u32 != 0 { + let entry = VIRTUAL_IPI_MAILBOX[target_cpu * 4].load(Ordering::Acquire) as usize; + let target_data = get_cpu_data(target_cpu); + let _lock = target_data.ctrl_lock.lock(); + if target_data.vcpu_state.is_stopped() { + debug!( + "loongarch64: guest boots vCPU{}(pCPU{}) through IPI, entry={:#x}", + target_guest_cpu, target_cpu, entry + ); + target_data.cpu_on_entry = entry; + target_data.vcpu_state.store(VcpuState::Ready); + send_event(target_cpu, SGI_IPI_ID as usize, IPI_EVENT_WAKEUP); + } else { + warn!( + "loongarch64: guest tried to boot running vCPU{}(pCPU{})", + target_guest_cpu, target_cpu + ); + } + } + + let pending = ipi_bits & !(SMP_BOOT_CPU as u32); + if pending == 0 { + if blocking { + fence(Ordering::SeqCst); + } + return Ok(()); + } + + // Match the in-kernel LoongArch IPI model: commit the status update before + // returning from the send write, merge repeated actions, and only notify on + // the zero-to-nonzero transition. Bit 31 does not wait for guest IPI_CLEAR. + let old = VIRTUAL_IPI_STATUS[target_cpu].fetch_or(pending, Ordering::AcqRel); + let new = old | pending; + if old != new { + trace_virtual_ipi_transition("set", target_cpu, pending, old, new, blocking); + } + if old == 0 && new != 0 { + sync_or_notify_virtual_ipi_line(target_cpu); + } + + if blocking { + // The emulated send is synchronous: the target status and event queue + // are globally visible before the trapped IOCSR write completes. + fence(Ordering::SeqCst); + } + + Ok(()) +} + +fn handle_ipi_any_send_mmio(mmio: &mut MMIOAccess) -> HvResult { + if !mmio.is_write { + mmio.value = 0; + return Ok(()); + } + if mmio.size != 4 { + warn!("loongarch64: unsupported guest IPI send size {}", mmio.size); + return Ok(()); + } + + let value = mmio.value as u32; + let ipi_id = value & 0x1f; + let target_guest_cpu = ((value >> 16) & 0x3ff) as usize; + let blocking = value & IOCSR_IPI_SEND_BLOCKING != 0; + virtual_ipi_send(target_guest_cpu, 1u32 << ipi_id, blocking) +} + +fn handle_ipi_mail_send_mmio(mmio: &mut MMIOAccess) -> HvResult { + if !mmio.is_write { + mmio.value = 0; + return Ok(()); + } + if mmio.size != 8 { + warn!( + "loongarch64: unsupported guest mailbox send size {}", + mmio.size + ); + return Ok(()); + } + + let value = mmio.value as u64; + let target_guest_cpu = ((value >> 16) & 0x3ff) as usize; + let box_half = ((value >> 2) & 0x7) as usize; + let mailbox = box_half / 2; + let is_high = (box_half & 1) != 0; + let data = (value >> 32) as u32; + + let Some(target_cpu) = guest_cpu_to_physical_cpu(target_guest_cpu) else { + warn!( + "loongarch64: invalid guest mailbox send target_vcpu={}, mailbox={}", + target_guest_cpu, mailbox + ); + return Ok(()); + }; + + if mailbox >= 4 { + warn!( + "loongarch64: invalid guest mailbox send target_vcpu={}, mailbox={}", + target_guest_cpu, mailbox + ); + return Ok(()); + } + + let slot = &VIRTUAL_IPI_MAILBOX[target_cpu * 4 + mailbox]; + if is_high { + let old = slot.load(Ordering::Acquire); + let new = (old & 0x0000_0000_ffff_ffff) | ((data as u64) << 32); + slot.store(new, Ordering::Release); + } else { + let old = slot.load(Ordering::Acquire); + let new = (old & 0xffff_ffff_0000_0000) | data as u64; + slot.store(new, Ordering::Release); + } + + Ok(()) +} + +fn handle_ipi_mmio(mmio: &mut MMIOAccess) -> HvResult { + let cpu_block = (mmio.address - offset(IPI_REG_BASE)) / 0x100; + let reg = (mmio.address - offset(IPI_REG_BASE)) % 0x100; + let guest_cpu = if cpu_block == 0 { + this_guest_cpu_id() + } else { + cpu_block + }; + let Some(cpu) = guest_cpu_to_physical_cpu(guest_cpu) else { + warn!( + "loongarch64: guest IPI register access for invalid vCPU {}", + guest_cpu + ); + if !mmio.is_write { + mmio.value = 0; + } + return Ok(()); + }; + + match reg { + IPI_STATUS => { + if !mmio.is_write { + mmio.value = VIRTUAL_IPI_STATUS[cpu].load(Ordering::Acquire) as usize; + if mmio.value != 0 { + debug!( + "loongarch64: guest cpu{} reads IPI_STATUS cpu{} = {:#x}", + this_guest_cpu_id(), + guest_cpu, + mmio.value + ); + } + } + } + IPI_ENABLE => { + if mmio.is_write { + VIRTUAL_IPI_ENABLE[cpu].store(mmio.value as u32, Ordering::Release); + } else { + mmio.value = VIRTUAL_IPI_ENABLE[cpu].load(Ordering::Acquire) as usize; + } + } + IPI_SET => { + if mmio.is_write { + virtual_ipi_send(guest_cpu, mmio.value as u32, false)?; + } else { + mmio.value = 0; + } + } + IPI_CLEAR => { + if mmio.is_write { + let mask = mmio.value as u32; + let old = VIRTUAL_IPI_STATUS[cpu].fetch_and(!mask, Ordering::AcqRel); + let new = old & !mask; + if old != new { + trace_virtual_ipi_transition("clear", cpu, mask, old, new, false); + } + if old != 0 && new == 0 { + sync_or_notify_virtual_ipi_line(cpu); + } + if old != 0 || mmio.value != 0 { + debug!( + "loongarch64: guest cpu{} clears IPI_STATUS cpu{}, mask={:#x}, old={:#x}", + this_guest_cpu_id(), + guest_cpu, + mmio.value, + old + ); + } + } else { + mmio.value = 0; + } + } + IPI_MAILBOX_BASE..=0x3f if reg < IPI_MAILBOX_BASE + IPI_MAILBOX_SIZE => { + let mailbox = (reg - IPI_MAILBOX_BASE) / 8; + if mmio.is_write { + VIRTUAL_IPI_MAILBOX[cpu * 4 + mailbox].store(mmio.value as u64, Ordering::Release); + } else { + mmio.value = + VIRTUAL_IPI_MAILBOX[cpu * 4 + mailbox].load(Ordering::Acquire) as usize; + } + } + _ => { + if !mmio.is_write { + mmio.value = 0; + } + } + } + + Ok(()) +} + fn handle_mmio_stats(mmio: &mut MMIOAccess) { let key = MMIOAccessKey { offset: mmio.address, @@ -644,27 +964,33 @@ pub fn loongarch_generic_mmio_handler(mmio: &mut MMIOAccess, arg: usize) -> HvRe let ret; - if is_in_mmio_range!(mmio.address, EXTIOI_MAP_CORE_BASE, EXTIOI_MAP_CORE_SIZE) { + if is_in_mmio_range!(mmio.address, IPI_SEND_BASE, IPI_SEND_SIZE) { + ret = handle_ipi_any_send_mmio(mmio); + } else if is_in_mmio_range!(mmio.address, IPI_MAIL_SEND_BASE, IPI_MAIL_SEND_SIZE) { + ret = handle_ipi_mail_send_mmio(mmio); + } else if is_in_mmio_range!(mmio.address, IPI_REG_BASE, IPI_REG_SIZE) { + ret = handle_ipi_mmio(mmio); + } else if is_in_mmio_range!(mmio.address, EXTIOI_MAP_CORE_BASE, EXTIOI_MAP_CORE_SIZE) { ret = handle_extioi_mapping_mmio(mmio, EXTIOI_MAP_CORE_BASE, EXTIOI_MAP_CORE_SIZE); } else if is_in_mmio_range!(mmio.address, EXTIOI_SR_CORE_BASE, EXTIOI_SR_CORE_SIZE) { ret = handle_extioi_status_mmio(mmio, EXTIOI_SR_CORE_BASE, EXTIOI_SR_CORE_SIZE); } else if is_in_mmio_range!(mmio.address, EXTIOI_ENABLE_BASE, EXTIOI_ENABLE_SIZE) { - if this_cpu_id() != 0 && mmio.is_write { - info!("nonroot's write to extioi enable regs, ignored"); + if !is_this_root_zone() && mmio.is_write { + debug!("nonroot's write to extioi enable regs, ignored"); return Ok(()); } else { ret = handle_generic_mmio(mmio, BASE_ADDR); } } else if is_in_mmio_range!(mmio.address, EXTIOI_BOUNCE_BASE, EXTIOI_BOUNCE_SIZE) { - if this_cpu_id() != 0 && mmio.is_write { - info!("nonroot's write to extioi bounce regs, ignored"); + if !is_this_root_zone() && mmio.is_write { + debug!("nonroot's write to extioi bounce regs, ignored"); return Ok(()); } else { ret = handle_generic_mmio(mmio, BASE_ADDR); } } else if is_in_mmio_range!(mmio.address, EXTIOI_NODE_SEL_BASE, EXTIOI_NODE_SEL_SIZE) { - if this_cpu_id() != 0 && mmio.is_write { - info!("nonroot's write to extioi node sel regs, ignored"); + if !is_this_root_zone() && mmio.is_write { + debug!("nonroot's write to extioi node sel regs, ignored"); return Ok(()); } else { ret = handle_generic_mmio(mmio, BASE_ADDR); @@ -700,7 +1026,10 @@ impl Zone { Ok(()) } - pub fn arch_zone_reset(&mut self, _config: &HvZoneConfig) -> HvResult { + pub fn arch_zone_reset(&mut self, config: &HvZoneConfig) -> HvResult { + for cpu in config.cpus().iter() { + reset_virtual_ipi_state(*cpu as usize); + } Ok(()) } } diff --git a/src/consts.rs b/src/consts.rs index 869ab32fc..6014fbcfe 100644 --- a/src/consts.rs +++ b/src/consts.rs @@ -28,9 +28,15 @@ use crate::{memory::addr::VirtAddr, platform::BOARD_NCPUS}; use core::arch::global_asm; /// Size of the hypervisor heap. +#[cfg(loongson_3a6000)] +pub const HV_HEAP_SIZE: usize = 8 * 1024 * 1024; // 8 MiB +#[cfg(not(loongson_3a6000))] pub const HV_HEAP_SIZE: usize = 1024 * 1024; // 1 MiB -/// Size of the hypervisor memory pool used for dynamic allocation. +/// Size of the hypervisor memory pool used for dynamic stage-2 page tables. +#[cfg(loongson_3a6000)] +pub const HV_MEM_POOL_SIZE: usize = 0x1000_0000; // 256 MiB +#[cfg(not(loongson_3a6000))] pub const HV_MEM_POOL_SIZE: usize = 64 * 1024 * 1024; // 64 MiB /// Size of the per-CPU data area, including stack and CPU-local data. diff --git a/src/device/irqchip/ls7a2000/mod.rs b/src/device/irqchip/ls7a2000/mod.rs index b34a766c5..fab96ebc5 100644 --- a/src/device/irqchip/ls7a2000/mod.rs +++ b/src/device/irqchip/ls7a2000/mod.rs @@ -24,19 +24,16 @@ use crate::{ register::{read_gcsr_estat, write_gcsr_estat}, }, consts::MAX_CPU_NUM, + event::{send_event, IPI_EVENT_VIRTIO_INJECT_IRQ}, + hypercall::SGI_IPI_ID, zone::Zone, }; use chip::*; -use loongArch64::register::tcfg; -use spin::Mutex; +use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; pub mod chip; pub fn primary_init_early() { - if this_cpu_id() != 0 { - info!("loongarch64: irqchip: primary_init_early: do nothing on secondary cpus"); - return; - } info!("loongarch64: irqchip: primary_init_early: checking iochip configs"); print_chip_info(); csr_disable_new_codec(); @@ -110,6 +107,94 @@ const INT_PERF: usize = 10; const INT_TIMER: usize = 11; const INT_IPI: usize = 12; +static GUEST_HWI_ASSERTED: [AtomicU32; MAX_CPU_NUM] = { + const C: AtomicU32 = AtomicU32::new(0); + [C; MAX_CPU_NUM] +}; + +const IRQ_TRACE_LIMIT: usize = 64; +static IRQ_TRACE_SEQ: AtomicUsize = AtomicUsize::new(0); + +pub fn set_guest_irq_line(cpu: usize, irq: usize, asserted: bool) -> bool { + if cpu >= MAX_CPU_NUM || !(INT_HWI0..=INT_HWI7).contains(&irq) { + error!( + "[HVDBG:irq-line] invalid cpu={} irq={} asserted={}", + cpu, irq, asserted + ); + return false; + } + + let mask = 1u32 << (irq - INT_HWI0); + let state = &GUEST_HWI_ASSERTED[cpu]; + let old = if asserted { + state.fetch_or(mask, Ordering::AcqRel) + } else { + state.fetch_and(!mask, Ordering::AcqRel) + }; + let new = if asserted { old | mask } else { old & !mask }; + if old == new { + return true; + } + + if IRQ_TRACE_SEQ.fetch_add(1, Ordering::Relaxed) < IRQ_TRACE_LIMIT { + info!( + "[HVDBG:irq-line] src_cpu={} dst_cpu={} irq={} level={} old={:#04x} new={:#04x}", + this_cpu_id(), + cpu, + irq, + asserted as u8, + old, + new + ); + } + if cpu == this_cpu_id() { + sync_guest_irqs(); + } else { + send_event(cpu, SGI_IPI_ID as usize, IPI_EVENT_VIRTIO_INJECT_IRQ); + } + true +} + +pub fn clear_guest_irq_lines(cpu: usize) { + if cpu >= MAX_CPU_NUM { + return; + } + let old = GUEST_HWI_ASSERTED[cpu].swap(0, Ordering::AcqRel); + if old == 0 { + return; + } + info!( + "[HVDBG:irq-line] src_cpu={} dst_cpu={} clear_all_for_reset old={:#04x}", + this_cpu_id(), + cpu, + old + ); + if cpu == this_cpu_id() { + sync_guest_irqs(); + } else { + send_event(cpu, SGI_IPI_ID as usize, IPI_EVENT_VIRTIO_INJECT_IRQ); + } +} + +pub fn sync_guest_irqs() { + let cpu = this_cpu_id(); + let desired_vip = GUEST_HWI_ASSERTED[cpu].load(Ordering::Acquire) as usize & 0xff; + use crate::arch::register::gintc; + let before = gintc::read(); + let old_vip = before.vip(); + let pip = before.pip(); + let hc = before.hc(); + if old_vip != desired_vip { + gintc::write_vip(desired_vip); + } + if IRQ_TRACE_SEQ.fetch_add(1, Ordering::Relaxed) < IRQ_TRACE_LIMIT { + info!( + "[HVDBG:gintc-sync] cpu={} old_vip={:#04x} new_vip={:#04x} pip={:#04x} hc={:#04x}", + cpu, old_vip, desired_vip, pip, hc + ); + } +} + /// inject irq to THIS cpu pub fn inject_irq(_irq: usize, is_hardware: bool) { debug!( @@ -122,41 +207,32 @@ pub fn inject_irq(_irq: usize, is_hardware: bool) { } let bit = 1 << _irq; if _irq >= INT_HWI0 && _irq <= INT_HWI7 { - // use gintc to inject - use crate::arch::register::gintc; - gintc::set_hwis(bit >> INT_HWI0); + set_guest_irq_line(this_cpu_id(), _irq, true); } else { // use gcsr to inject, just set the bit let mut gcsr_estat = read_gcsr_estat(); gcsr_estat |= bit; write_gcsr_estat(gcsr_estat); } - let mut status = GLOBAL_IRQ_INJECT_STATUS.lock(); - status.cpu_status[this_cpu_id()].status = InjectionStatus::Injecting; - - tcfg::set_en(true); // start timer to avoid endless timer injection - // please only enable this for debugging because it may cause overheads for realtime nonroots } -/// clear the injecting irq ctrl bit on THIS cpu -pub fn clear_hwi_injected_irq() { - use crate::arch::register::gintc; - gintc::set_hwis(0); - // gintc::set_hwip(0); - // gintc::set_hwic(0xff); - let mut gintc_raw = 0usize; - use core::arch::asm; - unsafe { - asm!("csrrd {0}, 0x52", out(reg) gintc_raw); +pub fn clear_injected_irq(_irq: usize) { + debug!("loongarch64: clear_injected_irq: _irq: {}", _irq); + if _irq > INT_IPI { + error!( + "loongarch64: clear_injected_irq: _irq > {}, not valid", + INT_IPI + ); + return; + } + let bit = 1 << _irq; + if _irq >= INT_HWI0 && _irq <= INT_HWI7 { + set_guest_irq_line(this_cpu_id(), _irq, false); + } else { + let mut gcsr_estat = read_gcsr_estat(); + gcsr_estat &= !bit; + write_gcsr_estat(gcsr_estat); } - debug!( - "loongarch64: clear_hwi_injected_irq: current gintc: {:#x}", - gintc_raw - ); - let mut status = GLOBAL_IRQ_INJECT_STATUS.lock(); - status.cpu_status[this_cpu_id()].status = InjectionStatus::Idle; - - tcfg::set_en(false); // stop timer } impl Zone { @@ -170,28 +246,3 @@ impl Zone { ); } } - -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum InjectionStatus { - Injecting, - Idle, -} - -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct PercpuInjectionStatus { - pub status: InjectionStatus, - pub irqs: [u32; 32], -} - -#[derive(Debug)] -pub struct GlobalInjectionStatus { - pub cpu_status: [PercpuInjectionStatus; MAX_CPU_NUM], -} - -pub static GLOBAL_IRQ_INJECT_STATUS: Mutex = - Mutex::new(GlobalInjectionStatus { - cpu_status: [PercpuInjectionStatus { - status: InjectionStatus::Idle, - irqs: [0; 32], - }; MAX_CPU_NUM], - }); diff --git a/src/device/virtio_trampoline.rs b/src/device/virtio_trampoline.rs index dfa2e6a4b..b6bec9b14 100644 --- a/src/device/virtio_trampoline.rs +++ b/src/device/virtio_trampoline.rs @@ -20,6 +20,8 @@ #![deny(unused_mut)] #![deny(unused)] +#[cfg(not(target_arch = "loongarch64"))] +use crate::device::irqchip::inject_irq; #[cfg(not(target_arch = "loongarch64"))] use crate::{ arch::cpu::get_target_cpu, @@ -27,9 +29,10 @@ use crate::{ hypercall::SGI_IPI_ID, }; use crate::{ - arch::cpu::this_cpu_id, consts::MAX_WAIT_TIMES, device::irqchip::inject_irq, error::HvResult, - memory::MMIOAccess, zone::this_zone_id, + arch::cpu::this_cpu_id, consts::MAX_WAIT_TIMES, error::HvResult, memory::MMIOAccess, + zone::this_zone_id, }; +#[cfg(not(target_arch = "loongarch64"))] use alloc::collections::BTreeMap; use core::{ fmt::{Debug, Formatter, Result}, @@ -43,6 +46,7 @@ use tock_registers::{ }; /// Save the irqs the virtio-device wants to inject. The format is >, and the first elem of List is the valid len of it. +#[cfg(not(target_arch = "loongarch64"))] pub static VIRTIO_IRQS: Mutex> = Mutex::new(BTreeMap::new()); // Controller of the shared memory the root linux's virtio device and hvisor shares. pub static VIRTIO_PCI_BRIDGE: Mutex = Mutex::new(VirtioPCIBridge::dummy()); @@ -157,6 +161,7 @@ pub fn check_need_wakeup_and_send_ipi(is_send_ipi: &mut bool) { /// When virtio req type is notify, root zone will send sgi to non root, \ /// and non root will call this function. +#[cfg(not(target_arch = "loongarch64"))] pub fn handle_virtio_irq() { let mut map = VIRTIO_IRQS.lock(); let irq_list = map.get_mut(&this_cpu_id()).unwrap(); diff --git a/src/event.rs b/src/event.rs index 7b147e161..05da65252 100644 --- a/src/event.rs +++ b/src/event.rs @@ -22,7 +22,7 @@ use crate::{ IPI_EVENT_UPDATE_HART_LINE, IPI_EVENT_VCPU_SUSPEND, MAX_CPU_NUM, }, cpu_data::{this_cpu_data, vcpu_suspend, CpuSet}, - device::{irqchip::inject_irq, virtio_trampoline::handle_virtio_irq}, + device::irqchip::inject_irq, platform::IRQ_WAKEUP_VIRTIO_DEVICE, }; #[cfg(virtio_pci)] @@ -31,6 +31,7 @@ use crate::{ platform::{IRQ_WAKEUP_VIRTIO_PCI_CONFIG, IRQ_WAKEUP_VIRTIO_PCI_DATA}, }; use alloc::{collections::VecDeque, vec::Vec}; +use core::sync::atomic::{AtomicUsize, Ordering}; use spin::Mutex; pub const IPI_EVENT_WAKEUP: usize = 0; @@ -42,39 +43,71 @@ pub const IPI_EVENT_VIRTIO_PCI_DATA: usize = 8; pub const IPI_EVENT_VIRTIO_PCI_DONE: usize = 9; #[percpu::def_percpu] -static PERCPU_EVENTS: Mutex> = Mutex::new(VecDeque::new()); +static PERCPU_EVENTS: Mutex = Mutex::new(EventQueueState::new()); + +struct EventQueueState { + queue: VecDeque, + doorbell_armed: bool, +} + +impl EventQueueState { + const fn new() -> Self { + Self { + queue: VecDeque::new(), + doorbell_armed: false, + } + } +} + +static EVENT_TRACE_SEQ: AtomicUsize = AtomicUsize::new(0); +const EVENT_TRACE_LIMIT: usize = 64; // The caller ensures the cpu_id is valid #[inline(always)] -fn get_percpu_events(cpu: usize) -> &'static Mutex> { +fn get_percpu_events(cpu: usize) -> &'static Mutex { unsafe { PERCPU_EVENTS.remote_ref_raw(cpu) } } -fn add_event(cpu: usize, event_id: usize) -> Option<()> { +fn add_event(cpu: usize, event_id: usize) -> Option { if cpu >= MAX_CPU_NUM { return None; } let mut e = get_percpu_events(cpu).lock(); if event_id == IPI_EVENT_SHUTDOWN { // If the event is shutdown, we need to clear all previous events, because shutdown will make cpu idle and won't process any events. - e.clear(); + e.queue.clear(); + } + e.queue.push_back(event_id); + let need_kick = !e.doorbell_armed; + e.doorbell_armed = true; + let seq = EVENT_TRACE_SEQ.fetch_add(1, Ordering::Relaxed); + if seq < EVENT_TRACE_LIMIT { + info!( + "[HVDBG:event-enqueue] seq={} src_cpu={} dst_cpu={} event={} armed={} queue_len={} kick={}", + seq, + this_cpu_data().id, + cpu, + event_id, + e.doorbell_armed, + e.queue.len(), + need_kick + ); } - e.push_back(event_id); - Some(()) + Some(need_kick) } pub fn fetch_event(cpu: usize) -> Option { if cpu >= MAX_CPU_NUM { return None; } - get_percpu_events(cpu).lock().pop_front() + get_percpu_events(cpu).lock().queue.pop_front() } pub fn dump_events() { for cpu in 0..MAX_CPU_NUM { let events = get_percpu_events(cpu).lock(); - if !events.is_empty() { - debug!("cpu {} events: {:?}", cpu, *events); + if !events.queue.is_empty() { + debug!("cpu {} events: {:?}", cpu, events.queue); } } } @@ -83,19 +116,25 @@ pub fn dump_cpu_events(cpu: usize) -> Vec { if cpu >= MAX_CPU_NUM { return Vec::new(); } - get_percpu_events(cpu).lock().iter().cloned().collect() + get_percpu_events(cpu) + .lock() + .queue + .iter() + .cloned() + .collect() } pub fn clear_events(cpu: usize) { if cpu >= MAX_CPU_NUM { return; } - get_percpu_events(cpu).lock().clear(); + let mut events = get_percpu_events(cpu).lock(); + events.queue.clear(); + events.doorbell_armed = false; } -pub fn check_events() -> bool { +fn handle_event(event: Option) -> bool { let cpu_data = this_cpu_data(); - let event = fetch_event(cpu_data.id); match event { Some(IPI_EVENT_WAKEUP) => { cpu_data.arch_cpu.run(); @@ -106,7 +145,10 @@ pub fn check_events() -> bool { false } Some(IPI_EVENT_VIRTIO_INJECT_IRQ) => { - handle_virtio_irq(); + #[cfg(target_arch = "loongarch64")] + crate::device::irqchip::ls7a2000::sync_guest_irqs(); + #[cfg(not(target_arch = "loongarch64"))] + crate::device::virtio_trampoline::handle_virtio_irq(); true } Some(IPI_EVENT_WAKEUP_VIRTIO_DEVICE) => { @@ -174,21 +216,45 @@ pub fn check_events() -> bool { } } +pub fn check_events() -> bool { + handle_event(fetch_event(this_cpu_data().id)) +} + +#[cfg(target_arch = "loongarch64")] +pub fn handle_next_loongarch_event(_ipi_int_id: usize) -> bool { + let cpu = this_cpu_data().id; + let (event, more_pending) = { + let mut state = get_percpu_events(cpu).lock(); + let event = state.queue.pop_front(); + let more_pending = !state.queue.is_empty(); + state.doorbell_armed = more_pending; + (event, more_pending) + }; + + if let Some(event_id) = event { + let seq = EVENT_TRACE_SEQ.fetch_add(1, Ordering::Relaxed); + if seq < EVENT_TRACE_LIMIT { + info!( + "[HVDBG:event-drain] seq={} cpu={} event={} more_pending={}", + seq, cpu, event_id, more_pending + ); + } + handle_event(Some(event_id)); + true + } else { + false + } +} + pub fn send_event(cpu_id: usize, ipi_int_id: usize, event_id: usize) { - // #[cfg(target_arch = "loongarch64")] - // { - // // block until the previous event is processed, which means - // // the target queue is empty - // while !fetch_event(cpu_id).is_none() {} - // debug!( - // "loongarch64:: send_event: cpu_id: {}, ipi_int_id: {}, event_id: {}", - // cpu_id, ipi_int_id, event_id - // ); - // } - /// Some arch need do something before send event. - /// Currently, we are not passing parameters, and we will modify the function signature later as needed. + // Some architectures need preparation before queueing an event. arch_prepare_send_event(cpu_id, ipi_int_id, event_id); - add_event(cpu_id, event_id); + let need_kick = add_event(cpu_id, event_id).unwrap_or(false); + #[cfg(target_arch = "loongarch64")] + if need_kick { + arch_send_event(cpu_id as _, ipi_int_id as _); + } + #[cfg(not(target_arch = "loongarch64"))] arch_send_event(cpu_id as _, ipi_int_id as _); } diff --git a/src/hypercall/mod.rs b/src/hypercall/mod.rs index ca4f33890..ee9b1b02c 100644 --- a/src/hypercall/mod.rs +++ b/src/hypercall/mod.rs @@ -18,22 +18,22 @@ use crate::arch::cpu::get_target_cpu; use crate::config::HvZoneConfig; -use crate::consts::{INVALID_ADDRESS, MAX_CPU_NUM, MAX_WAIT_TIMES, PAGE_SIZE}; +use crate::consts::{INVALID_ADDRESS, MAX_WAIT_TIMES, PAGE_SIZE}; use crate::cpu_data::{get_cpu_data, PerCpu, VcpuState}; use crate::device::virtio_trampoline::{ - VirtioPCIHypercallOp, MAX_DEVS, VIRTIO_BRIDGE, VIRTIO_IRQS, VIRTIO_PCI_BRIDGE, - VIRTIO_PCI_HYPERCALL_VERSION, + VirtioPCIHypercallOp, VIRTIO_BRIDGE, VIRTIO_PCI_BRIDGE, VIRTIO_PCI_HYPERCALL_VERSION, }; +#[cfg(not(target_arch = "loongarch64"))] +use crate::device::virtio_trampoline::{MAX_DEVS, VIRTIO_IRQS}; use crate::error::HvResult; use crate::pci::pci_config::GLOBAL_PCIE_LIST; use crate::zone::{ add_zone, all_zones_info, find_zone, is_this_root_zone, remove_zone, zone_create, ZoneInfo, }; -use crate::event::{ - send_event, IPI_EVENT_SHUTDOWN, IPI_EVENT_VIRTIO_INJECT_IRQ, IPI_EVENT_VIRTIO_PCI_DONE, - IPI_EVENT_WAKEUP, -}; +#[cfg(not(target_arch = "loongarch64"))] +use crate::event::IPI_EVENT_VIRTIO_INJECT_IRQ; +use crate::event::{send_event, IPI_EVENT_SHUTDOWN, IPI_EVENT_VIRTIO_PCI_DONE, IPI_EVENT_WAKEUP}; use core::convert::TryFrom; use numeric_enum_macro::numeric_enum; @@ -53,6 +53,9 @@ numeric_enum! { HvVirtioPCI = 7, } } +#[cfg(target_arch = "loongarch64")] +pub const SGI_IPI_ID: u64 = crate::arch::ipi::HVISOR_EVENT_DOORBELL as u64; +#[cfg(not(target_arch = "loongarch64"))] pub const SGI_IPI_ID: u64 = 7; pub type HyperCallResult = HvResult; @@ -89,15 +92,7 @@ impl<'a> HyperCall<'a> { HyperCallCode::HvZoneShutdown => self.hv_zone_shutdown(arg0), HyperCallCode::HvZoneList => self.hv_zone_list(&mut *(arg0 as *mut ZoneInfo), arg1), HyperCallCode::HvClearInjectIrq => { - use crate::consts::IPI_EVENT_CLEAR_INJECT_IRQ; - for i in 1..MAX_CPU_NUM { - // if target cpu status is not running, we skip it - if !get_cpu_data(i).vcpu_state.is_running() { - continue; - } - send_event(i, SGI_IPI_ID as _, IPI_EVENT_CLEAR_INJECT_IRQ); - } - HyperCallResult::Ok(0) + self.hv_virtio_deassert_irq(arg0 as usize, arg1 as usize) } HyperCallCode::HvIvcInfo => self.hv_ivc_info(arg0), HyperCallCode::HvConfigCheck => self.hv_zone_config_check(arg0 as *mut u64), @@ -147,6 +142,7 @@ impl<'a> HyperCall<'a> { ); } let mut res_agent = VIRTIO_BRIDGE.res_agent(); + #[cfg(not(target_arch = "loongarch64"))] let mut map_irq = VIRTIO_IRQS.lock(); while !res_agent.is_empty() { let (_res_front, irq_id, target_zone) = res_agent.peek_front(); @@ -158,27 +154,74 @@ impl<'a> HyperCall<'a> { } }; - let irq_list = map_irq.entry(target_cpu).or_insert([0; MAX_DEVS + 1]); - - self.wait_for_interrupt(irq_list); - if !irq_list[1..=irq_list[0] as usize].contains(&irq_id) { - let len = irq_list[0] as usize; - assert!(len + 1 < MAX_DEVS); - irq_list[len + 1] = irq_id; - irq_list[0] += 1; - send_event( - target_cpu as _, - SGI_IPI_ID as _, - IPI_EVENT_VIRTIO_INJECT_IRQ, + #[cfg(target_arch = "loongarch64")] + { + crate::device::irqchip::ls7a2000::set_guest_irq_line( + target_cpu, + irq_id as usize, + true, ); + res_agent.advance_front(); + continue; + } + + #[cfg(not(target_arch = "loongarch64"))] + { + let irq_list = map_irq.entry(target_cpu).or_insert([0; MAX_DEVS + 1]); + self.wait_for_interrupt(irq_list); + if !irq_list[1..=irq_list[0] as usize].contains(&irq_id) { + let len = irq_list[0] as usize; + assert!(len + 1 < MAX_DEVS); + irq_list[len + 1] = irq_id; + irq_list[0] += 1; + send_event( + target_cpu as _, + SGI_IPI_ID as _, + IPI_EVENT_VIRTIO_INJECT_IRQ, + ); + } } + #[cfg(not(target_arch = "loongarch64"))] res_agent.advance_front(); } drop(res_agent); HyperCallResult::Ok(0) } + fn hv_virtio_deassert_irq(&mut self, target_zone: usize, irq_id: usize) -> HyperCallResult { + if !is_this_root_zone() { + return hv_result_err!( + EPERM, + "Virtio deassert operation over non-root zones: unsupported!" + ); + } + if find_zone(target_zone).is_none() { + return hv_result_err!(EINVAL, format!("zone {} does not exist", target_zone)); + } + + let target_cpu = get_target_cpu(irq_id, target_zone); + #[cfg(target_arch = "loongarch64")] + { + if !crate::device::irqchip::ls7a2000::set_guest_irq_line(target_cpu, irq_id, false) { + return hv_result_err!( + EINVAL, + format!( + "invalid LoongArch guest HWI {} for zone {}", + irq_id, target_zone + ) + ); + } + return HyperCallResult::Ok(0); + } + + #[cfg(not(target_arch = "loongarch64"))] + hv_result_err!( + ENOSYS, + "per-IRQ Virtio deassert is not implemented on this architecture" + ) + } + pub fn hv_zone_start(&mut self, config: &HvZoneConfig, config_size: u64) -> HyperCallResult { let config_ipa = config as *const HvZoneConfig as u64; let config_pa = self.hv_get_real_pa(config_ipa); @@ -234,6 +277,7 @@ impl<'a> HyperCall<'a> { return hv_result_err!(EINVAL); } // avoid virtio daemon send sgi to the shutdowning zone + #[cfg(not(target_arch = "loongarch64"))] let mut map_irq = VIRTIO_IRQS.lock(); let zone = match find_zone(zone_id as _) { @@ -250,8 +294,11 @@ impl<'a> HyperCall<'a> { zone_w.cpu_set().iter().for_each(|cpu_id| { let _lock = get_cpu_data(cpu_id).ctrl_lock.lock(); get_cpu_data(cpu_id).cpu_on_entry = INVALID_ADDRESS; + #[cfg(target_arch = "loongarch64")] + crate::device::irqchip::ls7a2000::clear_guest_irq_lines(cpu_id); send_event(cpu_id, SGI_IPI_ID as _, IPI_EVENT_SHUTDOWN); // set the virtio irq list's len to 0 + #[cfg(not(target_arch = "loongarch64"))] if let Some(irq_list) = map_irq.get_mut(&cpu_id) { irq_list[0] = 0; } diff --git a/src/pci/config_accessors/loongarch64.rs b/src/pci/config_accessors/loongarch64.rs index cf60e4370..43a0ecc67 100644 --- a/src/pci/config_accessors/loongarch64.rs +++ b/src/pci/config_accessors/loongarch64.rs @@ -16,7 +16,7 @@ use alloc::sync::Arc; -use super::{PciConfigAccessor, PciConfigMmio}; +use super::{PciConfigAccessor, PciConfigMmio, PciRegion}; use crate::{ error::HvResult, @@ -40,60 +40,29 @@ impl RootComplex { // LoongArch PCIe accessor implementation #[derive(Debug)] pub struct LoongArchConfigAccessor { - cfg0: PciConfigAddress, - cfg1: PciConfigAddress, + cfg_base: PciConfigAddress, root_bus: u8, } impl LoongArchConfigAccessor { - pub fn new(cfg_base: PciConfigAddress, cfg_size: u64, root_bus: u8) -> Self { - let cfg_size_half = cfg_size / 2; - let cfg0 = cfg_base; - let cfg1 = cfg_base + cfg_size_half; - Self { - cfg0, - cfg1, - root_bus, + pub fn new(cfg_base: PciConfigAddress, _cfg_size: u64, root_bus: u8) -> Self { + Self { cfg_base, root_bus } + } + + fn cfg_map(&self, bdf: Bdf, offset: PciConfigAddress) -> PciConfigAddress { + let devfn = ((bdf.device() as PciConfigAddress) << 3) | bdf.function() as PciConfigAddress; + let mut addroff = (devfn << 8) | (offset & 0xff) | ((offset & 0xf00) << 16); + if bdf.bus() != self.root_bus { + addroff |= 1 << 28; + addroff |= (bdf.bus() as PciConfigAddress) << 16; } + self.cfg_base + addroff } } impl PciConfigAccessor for LoongArchConfigAccessor { fn get_pci_addr_base(&self, bdf: Bdf) -> HvResult { - let bus = bdf.bus() as PciConfigAddress; - let device = bdf.device() as PciConfigAddress; - let function = bdf.function() as PciConfigAddress; - let offset = 0; - - // Extract Offset[11:8] (bits 11-8 of offset) for bits 31-28 - let offset_high = (offset >> 8) & 0xf; - // Extract Offset[7:0] (bits 7-0 of offset) for bits 7-0 - let offset_low = offset & 0xff; - - let address = if bus == self.root_bus as PciConfigAddress { - // Type 0 format (Root Bus): - // Bits 31-28: Offset[11:8] - // Bits 27-16: Reserved (0) - // Bits 15-11: Device Number - // Bits 10-8: Function Number - // Bits 7-0: Offset[7:0] - self.cfg0 + ((offset_high << 24) | (device << 11) | (function << 8) | offset_low) - } else { - // Type 1 format (Other Bus): - // Bits 31-28: Offset[11:8] - // Bits 27-16: Bus Number - // Bits 15-11: Device Number - // Bits 10-8: Function Number - // Bits 7-0: Offset[7:0] - self.cfg1 - + ((offset_high << 24) - | (bus << 16) - | (device << 11) - | (function << 8) - | offset_low) - }; - - Ok(address) + Ok(self.cfg_map(bdf, 0)) } fn get_physical_address( @@ -102,39 +71,7 @@ impl PciConfigAccessor for LoongArchConfigAccessor { offset: PciConfigAddress, _parent_bus: u8, ) -> HvResult { - let bus = bdf.bus() as PciConfigAddress; - let device = bdf.device() as PciConfigAddress; - let function = bdf.function() as PciConfigAddress; - - // Extract Offset[11:8] (bits 11-8 of offset) for bits 31-28 - let offset_high = (offset >> 8) & 0xf; - // Extract Offset[7:0] (bits 7-0 of offset) for bits 7-0 - let offset_low = offset & 0xff; - - let address = if bus == self.root_bus as PciConfigAddress { - // Type 0 format (Root Bus): - // Bits 31-28: Offset[11:8] - // Bits 27-16: Reserved (0) - // Bits 15-11: Device Number - // Bits 10-8: Function Number - // Bits 7-0: Offset[7:0] - self.cfg0 + ((offset_high << 24) | (device << 11) | (function << 8) | offset_low) - } else { - // Type 1 format (Other Bus): - // Bits 31-28: Offset[11:8] - // Bits 27-16: Bus Number - // Bits 15-11: Device Number - // Bits 10-8: Function Number - // Bits 7-0: Offset[7:0] - self.cfg1 - + ((offset_high << 24) - | (bus << 16) - | (device << 11) - | (function << 8) - | offset_low) - }; - - Ok(address) + Ok(self.cfg_map(bdf, offset)) } fn skip_device(&self, _bdf: Bdf) -> bool { @@ -147,6 +84,61 @@ pub const LOONG_HT_PREFIX: u64 = 0xe00_0000_0000; impl PciConfigMmio { pub(crate) fn access(&self, offset: PciConfigAddress) -> *mut T { - (self.base + offset | HV_ADDR_PREFIX | LOONG_HT_PREFIX) as *mut T + let addroff = (offset & 0xff) | ((offset & 0xf00) << 16); + (self.base + addroff | HV_ADDR_PREFIX | LOONG_HT_PREFIX) as *mut T + } + + fn read_u32_aligned(&self, offset: PciConfigAddress) -> u32 { + unsafe { self.access::(offset & !0x3).read_volatile() } + } + + fn write_u32_aligned(&self, offset: PciConfigAddress, value: u32) { + unsafe { self.access::(offset & !0x3).write_volatile(value) } + } + + fn read_aligned_part(&self, offset: PciConfigAddress, size: u32) -> u32 { + let value = self.read_u32_aligned(offset); + let shift = ((offset & 0x3) * 8) as u32; + let mask = (1u32 << (size * 8)) - 1; + + (value >> shift) & mask + } + + fn write_aligned_part(&self, offset: PciConfigAddress, size: u32, value: u32) { + let shift = ((offset & 0x3) * 8) as u32; + let mask = !(((1u32 << (size * 8)) - 1) << shift); + let old = self.read_u32_aligned(offset); + let new = (old & mask) | (value << shift); + + self.write_u32_aligned(offset, new); + } +} + +impl PciRegion for PciConfigMmio { + fn read_u8(&self, offset: PciConfigAddress) -> HvResult { + Ok(self.read_aligned_part(offset, 1) as u8) + } + + fn write_u8(&self, offset: PciConfigAddress, value: u8) -> HvResult { + self.write_aligned_part(offset, 1, value as u32); + Ok(()) + } + + fn read_u16(&self, offset: PciConfigAddress) -> HvResult { + Ok(self.read_aligned_part(offset, 2) as u16) + } + + fn write_u16(&self, offset: PciConfigAddress, value: u16) -> HvResult { + self.write_aligned_part(offset, 2, value as u32); + Ok(()) + } + + fn read_u32(&self, offset: PciConfigAddress) -> HvResult { + Ok(self.read_u32_aligned(offset)) + } + + fn write_u32(&self, offset: PciConfigAddress, value: u32) -> HvResult { + self.write_u32_aligned(offset, value); + Ok(()) } } diff --git a/src/pci/config_accessors/mod.rs b/src/pci/config_accessors/mod.rs index 1a974546d..305a826cb 100644 --- a/src/pci/config_accessors/mod.rs +++ b/src/pci/config_accessors/mod.rs @@ -96,6 +96,7 @@ impl PciRegion for PciRegionMmio { } } +#[cfg(not(loongarch64_pcie))] impl PciRegion for PciConfigMmio { fn read_u8(&self, offset: PciConfigAddress) -> HvResult { unsafe { Ok(self.access::(offset).read_volatile() as u8) } diff --git a/src/pci/pci_handler.rs b/src/pci/pci_handler.rs index 71cc787a1..b1393f80e 100644 --- a/src/pci/pci_handler.rs +++ b/src/pci/pci_handler.rs @@ -2535,8 +2535,19 @@ fn set_pci_init_done(domain_id: u8) { pub fn mmio_vpci_direct_handler(mmio: &mut MMIOAccess, _base: usize) -> HvResult { let zone = this_zone(); - let offset = (mmio.address & 0xfff) as PciConfigAddress; - let base = mmio.address as PciConfigAddress - offset + _base as PciConfigAddress; + #[cfg(loongarch64_pcie)] + let (offset, base) = { + let access_addr = mmio.address as PciConfigAddress + _base as PciConfigAddress; + let offset = (access_addr & 0xff) | ((access_addr >> 16) & 0xf00); + let addroff = (offset & 0xff) | ((offset & 0xf00) << 16); + (offset, access_addr - addroff) + }; + #[cfg(not(loongarch64_pcie))] + let (offset, base) = { + let offset = (mmio.address & 0xfff) as PciConfigAddress; + let base = mmio.address as PciConfigAddress - offset + _base as PciConfigAddress; + (offset, base) + }; let mut is_dev_belong_to_zone = false; let dev: Option = { diff --git a/src/pci/pci_struct.rs b/src/pci/pci_struct.rs index 2ec58c853..788929978 100644 --- a/src/pci/pci_struct.rs +++ b/src/pci/pci_struct.rs @@ -26,7 +26,7 @@ use core::{ use spin::RwLock; use super::{ - config_accessors::{PciConfigAccessor, PciConfigMmio}, + config_accessors::{PciConfigAccessor, PciConfigMmio, PciRegion}, mem_alloc::BarAllocator, pci_access::{ Bar, EndpointField, EndpointHeader, HeaderType, PciBarRW, PciBridgeHeader, PciCommand, @@ -155,6 +155,9 @@ const PCI_EXP_TYPE_ROOT_PORT: u16 = 4; const PCI_EXP_TYPE_UPSTREAM: u16 = 5; const PCI_EXP_TYPE_DOWNSTREAM: u16 = 6; const PCI_EXP_TYPE_PCIE_BRIDGE: u16 = 8; +const PCIE_DEVICE_CAPABILITIES_2_OFFSET: PciConfigAddress = 0x24; +const PCIE_DEVICE_CONTROL_2_OFFSET: PciConfigAddress = 0x28; +const PCIE_ARI_FORWARDING: u32 = 1 << 5; pub(crate) const SRIOV_CAP_SIZE: PciConfigAddress = 0x40; const SRIOV_CTRL_OFFSET: PciConfigAddress = 0x08; @@ -164,9 +167,12 @@ const SRIOV_NUM_VFS_OFFSET: PciConfigAddress = 0x10; const SRIOV_FIRST_VF_OFFSET: PciConfigAddress = 0x14; const SRIOV_VF_STRIDE_OFFSET: PciConfigAddress = 0x16; const SRIOV_VF_DEVICE_ID_OFFSET: PciConfigAddress = 0x1a; +const SRIOV_SUPPORTED_PAGE_SIZE_OFFSET: PciConfigAddress = 0x1c; +const SRIOV_SYSTEM_PAGE_SIZE_OFFSET: PciConfigAddress = 0x20; pub(crate) const SRIOV_VF_BAR_OFFSET: PciConfigAddress = 0x24; pub(crate) const SRIOV_VF_BAR_END: PciConfigAddress = SRIOV_VF_BAR_OFFSET + 6 * 4; const SRIOV_CTRL_VF_ENABLE: u16 = 1 << 0; +const SRIOV_CTRL_ARI_CAPABLE_HIERARCHY: u16 = 1 << 4; #[derive(Clone, Copy, Eq, PartialEq, Default)] pub struct Bdf { @@ -800,6 +806,34 @@ impl VirtualPciConfigSpace { .find_map(|(offset, cap)| (cap.cap_type == cap_type).then_some(*offset)) } + fn sriov_write_system_page_size(&self, cap_offset: PciConfigAddress) -> HvResult<()> { + let supported = self + .backend + .read(cap_offset + SRIOV_SUPPORTED_PAGE_SIZE_OFFSET, 4)? as u32; + if supported == 0 { + return Ok(()); + } + + let page_size = supported & supported.wrapping_neg(); + self.backend.write( + cap_offset + SRIOV_SYSTEM_PAGE_SIZE_OFFSET, + 4, + page_size as usize, + ) + } + + fn sriov_latch_routing_fields( + &self, + cap_offset: PciConfigAddress, + num_vfs: u16, + ) -> HvResult<(u16, u16)> { + self.backend + .write(cap_offset + SRIOV_NUM_VFS_OFFSET, 2, num_vfs as usize)?; + let first_vf_offset = self.backend.read(cap_offset + SRIOV_FIRST_VF_OFFSET, 2)? as u16; + let vf_stride = self.backend.read(cap_offset + SRIOV_VF_STRIDE_OFFSET, 2)? as u16; + Ok((first_vf_offset, vf_stride)) + } + fn parse_sriov_vf_bars(&self, cap_offset: PciConfigAddress) -> HvResult { let mut bararr = Bar::default(); let mut slot = 0usize; @@ -892,7 +926,7 @@ impl VirtualPciConfigSpace { Ok(bararr) } - pub fn build_sriov_info(&mut self) -> HvResult<()> { + pub fn build_sriov_info(&mut self, upstream: Option<&Bridge>) -> HvResult<()> { let Some(cap_offset) = self.find_ext_cap_offset(ExtCapabilityType::SingleRootIov) else { return Ok(()); }; @@ -901,17 +935,79 @@ impl VirtualPciConfigSpace { .backend .read(cap_offset + SRIOV_INITIAL_VFS_OFFSET, 2)? as u16; let total_vfs = self.backend.read(cap_offset + SRIOV_TOTAL_VFS_OFFSET, 2)? as u16; - let first_vf_offset = self.backend.read(cap_offset + SRIOV_FIRST_VF_OFFSET, 2)? as u16; - let vf_stride = self.backend.read(cap_offset + SRIOV_VF_STRIDE_OFFSET, 2)? as u16; let vf_device_id = self .backend .read(cap_offset + SRIOV_VF_DEVICE_ID_OFFSET, 2)? as u16; - if total_vfs == 0 || first_vf_offset == 0 || vf_stride == 0 { + if total_vfs == 0 { return Ok(()); } + let ari_forwarding = match upstream { + Some(bridge) => bridge.enable_ari_forwarding()?, + None => false, + }; + + let mut ctrl = self.backend.read(cap_offset + SRIOV_CTRL_OFFSET, 2)? as u16; + if ctrl & SRIOV_CTRL_VF_ENABLE != 0 { + self.backend.write(cap_offset + SRIOV_CTRL_OFFSET, 2, 0)?; + ctrl = 0; + } + self.backend + .write(cap_offset + SRIOV_NUM_VFS_OFFSET, 2, 0)?; + + let sriov_ctrl = if ari_forwarding { + ctrl | SRIOV_CTRL_ARI_CAPABLE_HIERARCHY + } else { + ctrl & !SRIOV_CTRL_ARI_CAPABLE_HIERARCHY + }; + self.sriov_write_system_page_size(cap_offset)?; + self.backend + .write(cap_offset + SRIOV_CTRL_OFFSET, 2, sriov_ctrl as usize)?; + + let (first_vf_offset, vf_stride) = + self.sriov_latch_routing_fields(cap_offset, total_vfs)?; + if first_vf_offset == 0 || vf_stride == 0 { + warn!( + "SR-IOV on {:#?}: invalid routing fields offset={:#x} stride={} InitialVFs={} TotalVFs={} ari={} ctrl={:#x}", + self.bdf, + first_vf_offset, + vf_stride, + initial_vfs, + total_vfs, + ari_forwarding, + sriov_ctrl + ); + return Ok(()); + } + + let confirmed_ctrl = self.backend.read(cap_offset + SRIOV_CTRL_OFFSET, 2)? as u16; + if ari_forwarding && confirmed_ctrl & SRIOV_CTRL_ARI_CAPABLE_HIERARCHY == 0 { + warn!( + "SR-IOV ARI hierarchy bit did not stick for {:#?}: ctrl={:#x}", + self.bdf, confirmed_ctrl + ); + } else if !ari_forwarding { + warn!( + "SR-IOV on {:#?}: upstream ARI forwarding is not enabled; VF routing may use non-ARI offsets", + self.bdf + ); + } + + info!( + "SR-IOV on {:#?}: latched offset={:#x} stride={} NumVFs={} InitialVFs={} TotalVFs={} ari={} ctrl={:#x}", + self.bdf, + first_vf_offset, + vf_stride, + total_vfs, + initial_vfs, + total_vfs, + ari_forwarding, + confirmed_ctrl + ); + let vf_bars = self.parse_sriov_vf_bars(cap_offset)?; + let mut vf_bdfs = Vec::with_capacity(total_vfs as usize); for vf_index in 0..total_vfs { let route_offset = match first_vf_offset.checked_add(vf_stride.saturating_mul(vf_index)) @@ -934,13 +1030,10 @@ impl VirtualPciConfigSpace { access.set_bits(cap_offset as usize..(cap_offset as usize + 0x40)); }); - self.backend - .write(cap_offset + SRIOV_NUM_VFS_OFFSET, 2, total_vfs as usize)?; - let ctrl = self.backend.read(cap_offset + SRIOV_CTRL_OFFSET, 2)? as u16; self.backend.write( cap_offset + SRIOV_CTRL_OFFSET, 2, - (ctrl | SRIOV_CTRL_VF_ENABLE) as usize, + (sriov_ctrl | SRIOV_CTRL_VF_ENABLE) as usize, )?; let mut sriov_info = SriovInfo::new( @@ -953,7 +1046,7 @@ impl VirtualPciConfigSpace { vf_bars, vf_bdfs, ); - sriov_info.enabled_vfs = total_vfs; + sriov_info.enabled_vfs = sriov_info.vf_bdfs.len() as u16; self.sriov_info = Some(sriov_info); Ok(()) @@ -1731,7 +1824,10 @@ impl PciIterator { let _ = node.capability_enumerate(); node.ext_capability_enumerate(); #[cfg(sriov)] - let _ = node.build_sriov_info(); + { + let upstream = self.stack.last().filter(|b| !b.mmio.is_placeholder()); + let _ = node.build_sriov_info(upstream); + } // Build MSI/MSIX info once during device discovery node.build_msi_info(); @@ -1759,7 +1855,10 @@ impl PciIterator { let _ = node.capability_enumerate(); node.ext_capability_enumerate(); #[cfg(sriov)] - let _ = node.build_sriov_info(); + { + let upstream = self.stack.last().filter(|b| !b.mmio.is_placeholder()); + let _ = node.build_sriov_info(upstream); + } // Build MSI/MSIX info once during device discovery node.build_msi_info(); @@ -2027,15 +2126,8 @@ impl Iterator for PciIterator { #[cfg(not(no_pcie_bar_realloc))] let next_bus = parent.subordinate_bus + 1; - let bdf = Bdf::new(domain, next_bus, 0, 0); - // Use the current bridge's own bus as the immediate parent bus for - // CFG address computation. For multi-level bridges (especially on - // DWC), using parent.primary_bus (the upstream of the *parent*) - // would select the wrong CFG0/CFG1 path and fail to reach devices - // behind deeper bridges. - let immediate_parent_bus = parent.bus; Some(self.get_bridge().next_bridge( - self.address(immediate_parent_bus, bdf), + node.get_base(), node.has_only_one_child(), self.is_mulitple_function, self.function, @@ -2124,6 +2216,54 @@ impl Bridge { } } + fn find_pcie_cap_offset(&self) -> Option { + if self.mmio.is_placeholder() { + return None; + } + + let mut offset = (self.mmio.read_u8(0x34).ok()? as PciConfigAddress) & !0x3; + for _ in 0..48 { + if offset < 0x40 || offset >= CONFIG_LENTH { + return None; + } + + let cap_id = self.mmio.read_u8(offset).ok()? as PciConfigAddress; + if cap_id == CapabilityType::PciExpress.to_id() { + return Some(offset); + } + + let next = (self.mmio.read_u8(offset + 1).ok()? as PciConfigAddress) & !0x3; + if next == 0 || next == offset { + return None; + } + offset = next; + } + + None + } + + pub fn enable_ari_forwarding(&self) -> HvResult { + let Some(pcie_cap_offset) = self.find_pcie_cap_offset() else { + return Ok(false); + }; + + let dev_cap2 = self + .mmio + .read_u32(pcie_cap_offset + PCIE_DEVICE_CAPABILITIES_2_OFFSET)?; + if (dev_cap2 & PCIE_ARI_FORWARDING) == 0 { + return Ok(false); + } + + let dev_ctrl2_offset = pcie_cap_offset + PCIE_DEVICE_CONTROL_2_OFFSET; + let dev_ctrl2 = self.mmio.read_u16(dev_ctrl2_offset)?; + let ari_ctrl2 = dev_ctrl2 | (PCIE_ARI_FORWARDING as u16); + if ari_ctrl2 != dev_ctrl2 { + self.mmio.write_u16(dev_ctrl2_offset, ari_ctrl2)?; + } + + Ok(true) + } + pub fn update_bridge_bus(&mut self) { // Skip update for placeholder bridges (they don't have real mmio) if self.mmio.is_placeholder() { @@ -2195,10 +2335,15 @@ impl RootComplex { return Vec::new(); }; + if sriov_info.enabled_vfs == 0 { + return Vec::new(); + } + sriov_info .vf_bdfs .iter() .copied() + .take(sriov_info.enabled_vfs as usize) .enumerate() .filter_map(|(vf_index, vf_bdf)| { let backend_base = self diff --git a/src/zone.rs b/src/zone.rs index b9e68bc87..1e2efefac 100644 --- a/src/zone.rs +++ b/src/zone.rs @@ -119,6 +119,10 @@ pub struct ZoneInner { mmio: Vec, cpu_num: usize, cpu_set: CpuSet, + // Guest-visible CPU ids are not guaranteed to be dense or equal to pCPU ids. + // Keep explicit maps here; CpuSet only records physical CPU ownership. + guest_to_phys_cpu: [Option; MAX_CPU_NUM], + phys_to_guest_cpu: [Option; MAX_CPU_NUM], irq_bitmap: [u32; 1024 / 32], gpm: MemorySet, iommu_pt: Option>, @@ -174,6 +178,8 @@ impl ZoneInner { mmio: Vec::new(), cpu_num: 0, cpu_set: CpuSet::new(MAX_CPU_NUM as usize, 0), + guest_to_phys_cpu: [None; MAX_CPU_NUM], + phys_to_guest_cpu: [None; MAX_CPU_NUM], irq_bitmap: [0; 1024 / 32], iommu_pt: if cfg!(iommu) { Some(new_s2_memory_set()) @@ -276,6 +282,36 @@ impl ZoneInner { &mut self.cpu_set } + pub fn map_cpu(&mut self, guest_cpu: usize, phys_cpu: usize) { + if guest_cpu >= MAX_CPU_NUM || phys_cpu >= MAX_CPU_NUM { + warn!( + "ignore invalid CPU map guest_cpu={}, phys_cpu={}", + guest_cpu, phys_cpu + ); + return; + } + if let Some(old_phys_cpu) = self.guest_to_phys_cpu[guest_cpu] { + if self.phys_to_guest_cpu[old_phys_cpu] == Some(guest_cpu) { + self.phys_to_guest_cpu[old_phys_cpu] = None; + } + } + if let Some(old_guest_cpu) = self.phys_to_guest_cpu[phys_cpu] { + if self.guest_to_phys_cpu[old_guest_cpu] == Some(phys_cpu) { + self.guest_to_phys_cpu[old_guest_cpu] = None; + } + } + self.guest_to_phys_cpu[guest_cpu] = Some(phys_cpu); + self.phys_to_guest_cpu[phys_cpu] = Some(guest_cpu); + } + + pub fn guest_to_phys_cpu(&self, guest_cpu: usize) -> Option { + self.guest_to_phys_cpu.get(guest_cpu).copied().flatten() + } + + pub fn phys_to_guest_cpu(&self, phys_cpu: usize) -> Option { + self.phys_to_guest_cpu.get(phys_cpu).copied().flatten() + } + pub fn irq_bitmap(&self) -> &[u32; 1024 / 32] { &self.irq_bitmap } @@ -798,7 +834,7 @@ pub fn zone_create(config: &HvZoneConfig) -> HvResult> { zone.mmio_init(&config.arch_config); let mut cpu_num = 0; - for cpu_id in config.cpus().iter() { + for (guest_cpu, cpu_id) in config.cpus().iter().enumerate() { if let Some(existing_zone) = get_cpu_data(*cpu_id as _).zone.clone() { return hv_result_err!( EBUSY, @@ -809,7 +845,12 @@ pub fn zone_create(config: &HvZoneConfig) -> HvResult> { ) ); } - zone.write().cpu_set_mut().set_bit(*cpu_id as _); + let cpu_id = *cpu_id as usize; + let mut zone_inner = zone.write(); + zone_inner.cpu_set_mut().set_bit(cpu_id); + // The config lists physical CPUs, while guests use dense CPU ids from + // zero. Preserve that distinction for non-root SMP zones. + zone_inner.map_cpu(guest_cpu, cpu_id); cpu_num += 1; } zone.write().set_cpu_num(cpu_num); diff --git a/tools/kconfig/bootstrap_venv.sh b/tools/kconfig/bootstrap_venv.sh index a1830b9c3..9342100f9 100755 --- a/tools/kconfig/bootstrap_venv.sh +++ b/tools/kconfig/bootstrap_venv.sh @@ -11,9 +11,11 @@ if [[ ! -f "${REQ}" ]]; then exit 1 fi -if [[ -x "${VENV}/bin/python" ]] && "${VENV}/bin/python" -c "import kconfiglib" 2>/dev/null; then +if [[ -x "${VENV}/bin/python" ]] && "${VENV}/bin/python" -c 'import kconfiglib, sys; __import__("tomllib" if sys.version_info >= (3, 11) else "tomli")' 2>/dev/null; then exit 0 fi -python3 -m venv "${VENV}" +if [[ ! -x "${VENV}/bin/pip" ]]; then + python3 -m venv "${VENV}" +fi "${VENV}/bin/pip" install -q -r "${REQ}" diff --git a/tools/kconfig/kconfig_cli.py b/tools/kconfig/kconfig_cli.py index ffef3f699..a2964b240 100644 --- a/tools/kconfig/kconfig_cli.py +++ b/tools/kconfig/kconfig_cli.py @@ -7,9 +7,13 @@ import os import re import sys -import tomllib from pathlib import Path +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib + _KCONF_ARCH_TO_DIR: dict[str, str] = { "ARCH_AARCH64": "aarch64", "ARCH_RISCV64": "riscv64", diff --git a/tools/kconfig/requirements.txt b/tools/kconfig/requirements.txt index 3c91d8914..1e0d7bc4a 100644 --- a/tools/kconfig/requirements.txt +++ b/tools/kconfig/requirements.txt @@ -1,2 +1,3 @@ PySocks>=1.7.1 kconfiglib>=14.1.0 +tomli>=2.0.1; python_version < "3.11"