[REBASE & FF] Refactor Early Cpu Init

core/patina_internal_cpu/src/gdt.rs

@@ -6,10 +6,9 @@
 //!
 //! SPDX-License-Identifier: Apache-2.0
 //!
-#![cfg_attr(test, allow(dead_code))]
+#![cfg_attr(any(test, not(target_os = "uefi")), allow(dead_code))]
 #![cfg_attr(test, allow(unused_imports))]
 use core::ptr::{addr_of, addr_of_mut};
-use patina::SIZE_4GB;
 
 // The AP GDT only exists for the `mp` module, which needs `alloc`.
 #[cfg(any(feature = "alloc", test))]
@@ -235,45 +234,6 @@ pub(crate) fn descriptor() -> DescriptorTablePointer {
     }
 }
 
-#[cfg_attr(coverage, coverage(off))]
-/// Initializes the GDT from a fixed descriptor set and loads it.
-pub fn init() {
-    let gdt_ptr = GDT.as_ptr() as usize;
-    assert!(gdt_ptr < SIZE_4GB, "GDT above 4GB, MP services will fail");
-
-    let gdtr = descriptor();
-
-    // SAFETY: We are constructing a well known GDT that maps all segments in a flat map
-    unsafe {
-        core::arch::asm!("lgdt [{}]", in(reg) &raw const gdtr, options(nostack, preserves_flags));
-
-        // Reload CS via a far return
-        core::arch::asm!(
-            "push {sel}",
-            "lea {tmp}, [rip + 2f]",
-            "push {tmp}",
-            "retfq",
-            "2:",
-            sel = in(reg) u64::from(CODE_SELECTOR),
-            tmp = lateout(reg) _,
-            options(preserves_flags),
-        );
-
-        // These segments need to be valid, but can be all the same. Program them to the same GDT entry,
-        // following what the C codebase does, as these are unused in long mode.
-        core::arch::asm!("mov ss, {0:x}", in(reg) DATA_SELECTOR, options(nostack, preserves_flags));
-        core::arch::asm!("mov ds, {0:x}", in(reg) DATA_SELECTOR, options(nostack, preserves_flags));
-        core::arch::asm!("mov es, {0:x}", in(reg) DATA_SELECTOR, options(nostack, preserves_flags));
-        core::arch::asm!("mov fs, {0:x}", in(reg) DATA_SELECTOR, options(nostack, preserves_flags));
-        core::arch::asm!("mov gs, {0:x}", in(reg) DATA_SELECTOR, options(nostack, preserves_flags));
-
-        // Load TSS
-        core::arch::asm!("ltr {0:x}", in(reg) TSS_SELECTOR, options(nostack, preserves_flags));
-    }
-
-    log::info!("Loaded GDT @ {:p}", GDT.as_ptr());
-}
-
 #[cfg(test)]
 #[cfg_attr(coverage, coverage(off))]
 mod tests {

core/patina_internal_cpu/src/interrupts/aarch64/interrupt_manager.rs

@@ -14,7 +14,8 @@ use patina::{
 use patina_paging::PageTable;
 
 use crate::interrupts::{
-    EfiExceptionInfoDump, EfiSystemContext, HandlerType, InterruptManager, aarch64::ExceptionContextAArch64,
+    EfiExceptionInfoDump, EfiSystemContext, ExceptionType, HandlerType, InterruptHandler, InterruptManager,
+    aarch64::ExceptionContextAArch64,
 };
 
 cfg_if::cfg_if! {
@@ -48,13 +49,20 @@ impl InterruptsAarch64 {
     /// for interrupts and exceptions to be taken. This routine may install some
     /// architecture specific default handlers for exceptions.
     ///
-    pub fn initialize(&mut self) -> Result<(), EfiError> {
-        // Initialize exception entrypoint
-        initialize_exception()?;
-
+    /// Default and platform handlers are registered before activating the exception vector table.
+    pub fn initialize(
+        &mut self,
+        exception_handlers: &[(ExceptionType, &'static dyn InterruptHandler)],
+    ) -> Result<(), EfiError> {
         self.register_exception_handler(0, HandlerType::UefiRoutine(synchronous_exception_handler))
             .expect("Failed to install default exception handler!");
 
+        for &(exception_type, handler) in exception_handlers {
+            self.register_exception_handler(exception_type, HandlerType::Handler(handler))?;
+        }
+
+        initialize_exception()?;
+
         Ok(())
     }
 }

core/patina_internal_cpu/src/interrupts/stub.rs

@@ -9,7 +9,7 @@
 
 use patina::{error::EfiError, pi::protocol::cpu_arch::EfiSystemContext};
 
-use crate::interrupts::InterruptManager;
+use crate::interrupts::{ExceptionType, HandlerType, InterruptHandler, InterruptManager};
 
 /// Null implementation of the `EfiSystemContextFactory` and `EfiExceptionInfoDump` traits.
 #[derive(Debug)]
@@ -38,8 +38,14 @@ impl InterruptsStub {
         Self {}
     }
 
-    /// A do-nothing initialization function for the null implementation.
-    pub fn initialize(&mut self) -> Result<(), EfiError> {
+    /// Registers supplied platform handlers without initializing hardware.
+    pub fn initialize(
+        &mut self,
+        exception_handlers: &[(ExceptionType, &'static dyn InterruptHandler)],
+    ) -> Result<(), EfiError> {
+        for &(exception_type, handler) in exception_handlers {
+            self.register_exception_handler(exception_type, HandlerType::Handler(handler))?;
+        }
         Ok(())
     }
 }

core/patina_internal_cpu/src/interrupts/x64.rs

@@ -11,6 +11,9 @@ use crate::{interrupts::ExceptionContextX64, log_registers};
 use patina::pi::protocol::cpu_arch::EfiSystemContext;
 use patina_stacktrace::{StackFrame, StackTrace};
 
+#[cfg_attr(coverage, coverage(off))]
+#[cfg(all(target_arch = "x86_64", target_os = "uefi"))]
+mod apic;
 #[cfg(target_arch = "x86_64")]
 pub(crate) mod idt;
 mod interrupt_manager;

core/patina_internal_cpu/src/interrupts/x64/apic.rs

@@ -0,0 +1,98 @@
+//! X64 APIC initialization.
+//!
+//! ## License
+//!
+//! Copyright (c) Microsoft Corporation.
+//! SPDX-License-Identifier: Apache-2.0
+//!
+
+use patina::{
+    arch::x64::{read_msr, write_msr},
+    mmio::{UniqueMmioPointer, fields::ReadPureWrite},
+};
+use patina::{bit, error::EfiError};
+
+const APIC_BASE_MASK: u64 = 0x000F_FFFF_FFFF_F000;
+const APIC_ENABLE: u64 = bit!(11);
+const X2APIC_ENABLE: u64 = bit!(10);
+const SVR_OFFSET: usize = 0xF0;
+const LINT0_OFFSET: usize = 0x350;
+const LINT1_OFFSET: usize = 0x360;
+const SPURIOUS_VECTOR_MASK: u32 = 0xFF;
+const SOFTWARE_ENABLE: u32 = bit!(8);
+const DELIVERY_MODE_MASK: u32 = bit!(8) | bit!(9) | bit!(10);
+const ACTIVE_LOW: u32 = bit!(13);
+const LEVEL_TRIGGERED: u32 = bit!(15);
+const MASKED: u32 = bit!(16);
+const EXTINT: u32 = bit!(8) | bit!(9) | bit!(10);
+const NMI: u32 = bit!(10);
+
+const APIC_BASE_MSR: u32 = 0x1B;
+const X2APIC_MSR_BASE: u32 = 0x800;
+
+#[derive(Debug, PartialEq)]
+enum ApicMode {
+    XApic(usize),
+    X2Apic,
+}
+
+impl ApicMode {
+    fn from_base(apic_base: u64) -> Result<Self, EfiError> {
+        if apic_base & APIC_ENABLE == 0 {
+            return Err(EfiError::Unsupported);
+        }
+        if apic_base & X2APIC_ENABLE != 0 {
+            Ok(Self::X2Apic)
+        } else {
+            let base = (apic_base & APIC_BASE_MASK) as usize;
+            if base == 0 {
+                return Err(EfiError::Unsupported);
+            }
+            Ok(Self::XApic(base))
+        }
+    }
+}
+
+fn program_virtual_wire_mode(mut update: impl FnMut(usize, u32, u32)) {
+    update(SVR_OFFSET, SPURIOUS_VECTOR_MASK, SOFTWARE_ENABLE | 0x0F);
+    let lint_mask = DELIVERY_MODE_MASK | ACTIVE_LOW | LEVEL_TRIGGERED | MASKED;
+    update(LINT0_OFFSET, lint_mask, EXTINT);
+    update(LINT1_OFFSET, lint_mask, NMI);
+}
+
+fn apic_base_msr_supported(features: u32) -> bool {
+    let required_features = bit!(5) | bit!(9);
+    features & required_features == required_features
+}
+
+pub(super) fn initialize_virtual_wire_mode() -> Result<(), EfiError> {
+    #[allow(unused_unsafe)]
+    // SAFETY: It is safe to read CPUID leaf 1
+    let cpu_features = unsafe { core::arch::x86_64::__cpuid(1) };
+    if !apic_base_msr_supported(cpu_features.edx) {
+        return Err(EfiError::Unsupported);
+    }
+
+    // SAFETY: It is safe to read this MSR once we have confirmed support for it
+    let mode = ApicMode::from_base(unsafe { read_msr(APIC_BASE_MSR) })?;
+    program_virtual_wire_mode(|offset, clear, set| match mode {
+        ApicMode::X2Apic => {
+            let msr = X2APIC_MSR_BASE + (offset >> 4) as u32;
+            // SAFETY: The enabled x2APIC exposes SVR and LINT0/1 at these architectural MSRs.
+            let value = unsafe { read_msr(msr) } as u32;
+            core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst);
+            // SAFETY: Only the documented SVR and LINT configuration fields are changed.
+            unsafe { write_msr(msr, u64::from((value & !clear) | set)) };
+        }
+        ApicMode::XApic(base) => {
+            let address = core::ptr::NonNull::new((base + offset) as *mut ReadPureWrite<u32>)
+                .expect("Validated nonzero APIC base");
+            // SAFETY: The enabled xAPIC's base MSR identifies its mapped register page. CPU
+            // initialization exclusively accesses these aligned SVR and LINT registers.
+            let mut register = unsafe { UniqueMmioPointer::new(address) };
+            let value = register.read();
+            register.write((value & !clear) | set);
+        }
+    });
+    Ok(())
+}

core/patina_internal_cpu/src/interrupts/x64/idt.rs

@@ -104,7 +104,7 @@ fn get_vector_address(index: usize) -> u64 {
 static IDT: StaticIdt = StaticIdt(UnsafeCell::new(Idt::filled(IdtEntry::empty())));
 
 #[cfg(target_os = "uefi")]
-pub fn initialize_idt() {
+pub(crate) fn prepare_idt() -> crate::gdt::DescriptorTablePointer {
     let cs = crate::gdt::CODE_SELECTOR;
     // SAFETY: There is only path to access the IDT and it is not possible to have concurrent access.
     let idt = unsafe { &mut *IDT.0.get() };
@@ -118,8 +118,5 @@ pub fn initialize_idt() {
     }
 
     assert!((IDT.0.get() as usize) < SIZE_4GB, "IDT above 4GB, MP services will fail");
-    let idtr = idt.descriptor();
-    // SAFETY: Loading our fully initialized IDT.
-    unsafe { core::arch::asm!("lidt [{}]", in(reg) core::ptr::addr_of!(idtr), options(nostack)) };
-    log::info!("Loaded IDT");
+    idt.descriptor()
 }

core/patina_internal_cpu/src/interrupts/x64/interrupt_manager.rs

@@ -18,7 +18,9 @@ use patina_mtrr::{Mtrr, error::MtrrError};
 use patina_paging::PageTable;
 use patina_stacktrace::{StackFrame, StackTrace};
 
-use crate::interrupts::{EfiExceptionInfoDump, HandlerType, InterruptManager, x64::ExceptionContextX64};
+use crate::interrupts::{
+    EfiExceptionInfoDump, ExceptionType, HandlerType, InterruptHandler, InterruptManager, x64::ExceptionContextX64,
+};
 
 /// X64 Implementation of the `InterruptManager`.
 ///
@@ -40,11 +42,12 @@ impl InterruptsX64 {
     /// for interrupts and exceptions to be taken. This routine may install some
     /// architecture specific default handlers for exceptions.
     ///
-    pub fn initialize(&mut self) -> Result<(), EfiError> {
-        // Initialize the IDT.
-        #[cfg(target_os = "uefi")]
-        crate::interrupts::x64::idt::initialize_idt();
-
+    /// Default and platform handlers are registered before activating the GDT and IDT.
+    /// Virtual-wire mode is then programmed on non-CC guests.
+    pub fn initialize(
+        &mut self,
+        exception_handlers: &[(ExceptionType, &'static dyn InterruptHandler)],
+    ) -> Result<(), EfiError> {
         // Register some default handlers.
         self.register_exception_handler(8, HandlerType::UefiRoutine(double_fault_handler))
             .expect("Failed to install default exception handler!");
@@ -53,12 +56,75 @@ impl InterruptsX64 {
         self.register_exception_handler(14, HandlerType::UefiRoutine(page_fault_handler))
             .expect("Failed to install default exception handler!");
 
+        for &(exception_type, handler) in exception_handlers {
+            self.register_exception_handler(exception_type, HandlerType::Handler(handler))?;
+        }
+        #[cfg(target_os = "uefi")]
+        {
+            patina::arch::disable_interrupts();
+            descriptor_table_install();
+            patina::arch::enable_interrupts();
+            self.initialize_virtual_wire_mode()?;
+        }
+
         Ok(())
     }
+
+    /// Programs virtual-wire mode on non-CC guests after platform exception handlers are installed. CC guests do not
+    /// require this and may fail if it is attempted. If this function is called from a CC guest it will simply return
+    /// `Ok()`.
+    #[cfg(target_os = "uefi")]
+    fn initialize_virtual_wire_mode(&mut self) -> Result<(), EfiError> {
+        if patina::arch::is_cc_guest() {
+            return Ok(());
+        }
+        super::apic::initialize_virtual_wire_mode()
+    }
 }
 
 impl InterruptManager for InterruptsX64 {}
 
+#[cfg_attr(coverage, coverage(off))]
+#[cfg(target_os = "uefi")]
+/// Prepares both descriptor tables, loads the GDT, reloads segments and the TSS, then loads the IDT.
+fn descriptor_table_install() {
+    let gdtr = crate::gdt::descriptor();
+    let gdt_ptr = gdtr.base as *const u64;
+    assert!((gdt_ptr as usize) < patina::SIZE_4GB, "GDT above 4GB, MP services will fail");
+
+    let idtr = super::idt::prepare_idt();
+
+    // SAFETY: Both descriptors reference fully initialized static tables. The GDT defines
+    // the code, data, and TSS selectors used below and by the IDT. Initialization is exclusive.
+    unsafe {
+        core::arch::asm!(
+            "lgdt [{gdtr}]",
+            "push {sel}",
+            "lea {tmp}, [rip + 2f]",
+            "push {tmp}",
+            "retfq",
+            "2:",
+            "mov ss, {data:x}",
+            "mov ds, {data:x}",
+            "mov es, {data:x}",
+            "mov fs, {data:x}",
+            "mov gs, {data:x}",
+            "ltr {tss:x}",
+            "lidt [{idtr}]",
+            gdtr = in(reg) &raw const gdtr,
+            idtr = in(reg) &raw const idtr,
+            sel = in(reg) u64::from(crate::gdt::CODE_SELECTOR),
+            data = in(reg) crate::gdt::DATA_SELECTOR,
+            tss = in(reg) crate::gdt::TSS_SELECTOR,
+            tmp = out(reg) _,
+            options(preserves_flags),
+        );
+    }
+
+    log::info!("Loaded GDT @ {gdt_ptr:p}");
+    log::info!("Loaded IDT");
+}
+
 #[cfg_attr(coverage, coverage(off))]
 /// Default handler for double faults.
 extern "efiapi" fn double_fault_handler(_exception_type: isize, context: EfiSystemContext) {
@@ -262,8 +328,32 @@ mod test {
     #[serial(exception_handlers)]
     fn test_interrupts_x64() {
         let mut interrupts = InterruptsX64::new();
-        assert!(interrupts.initialize().is_ok());
+        assert!(interrupts.initialize(&[]).is_ok());
+        assert!(interrupts.unregister_exception_handler(8).is_ok());
         assert!(interrupts.unregister_exception_handler(13).is_ok());
         assert!(interrupts.unregister_exception_handler(14).is_ok());
     }
+
+    #[test]
+    #[serial(exception_handlers)]
+    fn test_interrupts_x64_platform_handlers() {
+        struct PlatformHandler;
+
+        impl InterruptHandler for PlatformHandler {
+            fn handle_interrupt(
+                &'static self,
+                _exception_type: ExceptionType,
+                _context: &mut crate::interrupts::ExceptionContext,
+            ) {
+            }
+        }
+
+        static HANDLER: PlatformHandler = PlatformHandler;
+        let mut interrupts = InterruptsX64::new();
+        assert!(interrupts.initialize(&[(20, &HANDLER)]).is_ok());
+        assert!(interrupts.unregister_exception_handler(20).is_ok());
+        for exception_type in [8, 13, 14] {
+            assert!(interrupts.unregister_exception_handler(exception_type).is_ok());
+        }
+    }
 }