diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f172bdc92a..1e43c75e63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: id: cache-kvm-unit-test with: path: tests/kvm-unit-tests/ - key: ${{ runner.os }}-kvm-unit-test + key: ${{ runner.os }}-kvm-unit-test-${{ hashFiles('tests/kvm-unit-tests/**') }} - name: Build kvm-unit-test if: steps.cache-kvm-unit-test.outputs.cache-hit != 'true' @@ -74,13 +74,15 @@ jobs: tests/kvm-unit-tests/run.mjs tests/kvm-unit-tests/x86/taskswitch2.flat tests/kvm-unit-tests/run.mjs tests/kvm-unit-tests/x86/realmode.flat tests/kvm-unit-tests/run.mjs tests/kvm-unit-tests/x86/pat.flat + tests/kvm-unit-tests/run.mjs tests/kvm-unit-tests/x86/nx.flat + DISABLE_JIT=1 tests/kvm-unit-tests/run.mjs tests/kvm-unit-tests/x86/nx.flat - name: Fetch namsmtests cache uses: actions/cache@v5 id: cache-nasmtests with: path: tests/nasm/build/ - key: ${{ runner.os }}-nasmtests + key: ${{ runner.os }}-nasmtests-${{ hashFiles('gen/x86_table.js', 'tests/nasm/*.js', 'tests/nasm/*.asm', 'tests/nasm/*.inc', 'tests/nasm/gdb-extract-def') }} - name: Run nasmtests run: MAX_PARALLEL_TESTS=1 make nasmtests diff --git a/Makefile b/Makefile index 576f02902b..207179056a 100644 --- a/Makefile +++ b/Makefile @@ -338,6 +338,7 @@ kvm-unit-test: build/v86-debug.wasm tests/kvm-unit-tests/run.mjs tests/kvm-unit-tests/x86/taskswitch2.flat tests/kvm-unit-tests/run.mjs tests/kvm-unit-tests/x86/realmode.flat tests/kvm-unit-tests/run.mjs tests/kvm-unit-tests/x86/pat.flat + tests/kvm-unit-tests/run.mjs tests/kvm-unit-tests/x86/nx.flat kvm-unit-test-release: build/libv86.mjs build/v86.wasm tests/kvm-unit-tests/build.sh @@ -345,6 +346,7 @@ kvm-unit-test-release: build/libv86.mjs build/v86.wasm TEST_RELEASE_BUILD=1 tests/kvm-unit-tests/run.mjs tests/kvm-unit-tests/x86/taskswitch2.flat TEST_RELEASE_BUILD=1 tests/kvm-unit-tests/run.mjs tests/kvm-unit-tests/x86/realmode.flat TEST_RELEASE_BUILD=1 tests/kvm-unit-tests/run.mjs tests/kvm-unit-tests/x86/pat.flat + TEST_RELEASE_BUILD=1 tests/kvm-unit-tests/run.mjs tests/kvm-unit-tests/x86/nx.flat expect-tests: build/v86-debug.wasm build/libwabt.cjs make -C tests/expect/tests diff --git a/src/cpu.js b/src/cpu.js index 80720a0501..61cec32434 100644 --- a/src/cpu.js +++ b/src/cpu.js @@ -207,6 +207,7 @@ export function CPU(bus, wm, stop_idling) this.dreg = view(Int32Array, memory, 684, 8); this.reg_pdpte = view(Int32Array, memory, 968, 8); + this.efer = view(Uint32Array, memory, 1280, 2); this.pat = view(Uint32Array, memory, 1288, 2); this.svga_dirty_bitmap_min_offset = view(Uint32Array, memory, 716, 1); @@ -571,6 +572,7 @@ CPU.prototype.get_state = function() state[89] = this.devices.vmware; state[90] = this.devices.parallel0; state[91] = this.devices.parallel1; + state[92] = this.efer; state[93] = this.pat; return state; @@ -690,6 +692,8 @@ CPU.prototype.set_state = function(state) this.dreg.set(state[41]); state[42] && this.reg_pdpte.set(state[42]); this.pat.set(state[93] || [0x00070406, 0x00070406]); + this.efer.fill(0); + state[92] && this.efer.set(state[92]); this.set_tsc(state[43][0], state[43][1]); diff --git a/src/rust/codegen.rs b/src/rust/codegen.rs index fad00105cb..2d5c2ffe67 100644 --- a/src/rust/codegen.rs +++ b/src/rust/codegen.rs @@ -1,6 +1,6 @@ use crate::cpu::cpu::{ tlb_data, FLAG_CARRY, FLAG_OVERFLOW, FLAG_SIGN, FLAG_ZERO, OPSIZE_16, OPSIZE_32, OPSIZE_8, - TLB_GLOBAL, TLB_HAS_CODE, TLB_NO_USER, TLB_READONLY, TLB_VALID, + TLB_GLOBAL, TLB_HAS_CODE, TLB_NO_EXEC, TLB_NO_USER, TLB_READONLY, TLB_VALID, }; use crate::cpu::global_pointers; use crate::cpu::memory; @@ -671,6 +671,7 @@ fn gen_safe_read( & !TLB_READONLY & !TLB_GLOBAL & !TLB_HAS_CODE + & !TLB_NO_EXEC & !(if ctx.cpu.cpl3() { 0 } else { TLB_NO_USER })) as i32, ); ctx.builder.and_i32(); @@ -882,8 +883,10 @@ fn gen_safe_write( .load_aligned_i32(unsafe { &tlb_data[0] as *const i32 as u32 }); let entry_local = ctx.builder.tee_new_local(); - ctx.builder - .const_i32((0xFFF & !TLB_GLOBAL & !(if ctx.cpu.cpl3() { 0 } else { TLB_NO_USER })) as i32); + ctx.builder.const_i32( + (0xFFF & !TLB_GLOBAL & !TLB_NO_EXEC & !(if ctx.cpu.cpl3() { 0 } else { TLB_NO_USER })) + as i32, + ); ctx.builder.and_i32(); ctx.builder.const_i32(TLB_VALID as i32); @@ -1034,8 +1037,10 @@ pub fn gen_safe_read_write( .load_aligned_i32(unsafe { &tlb_data[0] as *const i32 as u32 }); let entry_local = ctx.builder.tee_new_local(); - ctx.builder - .const_i32((0xFFF & !TLB_GLOBAL & !(if ctx.cpu.cpl3() { 0 } else { TLB_NO_USER })) as i32); + ctx.builder.const_i32( + (0xFFF & !TLB_GLOBAL & !TLB_NO_EXEC & !(if ctx.cpu.cpl3() { 0 } else { TLB_NO_USER })) + as i32, + ); ctx.builder.and_i32(); ctx.builder.const_i32(TLB_VALID as i32); diff --git a/src/rust/cpu/cpu.rs b/src/rust/cpu/cpu.rs index 6bc928082a..7b03975311 100644 --- a/src/rust/cpu/cpu.rs +++ b/src/rust/cpu/cpu.rs @@ -165,6 +165,7 @@ pub const PAGE_TABLE_ACCESSED_MASK: i32 = 1 << 5; pub const PAGE_TABLE_DIRTY_MASK: i32 = 1 << 6; pub const PAGE_TABLE_PSE_MASK: i32 = 1 << 7; pub const PAGE_TABLE_GLOBAL_MASK: i32 = 1 << 8; +pub const PAGE_TABLE_NX_MASK: u64 = 1 << 63; pub const MMAP_BLOCK_BITS: i32 = 17; pub const MMAP_BLOCK_SIZE: i32 = 1 << MMAP_BLOCK_BITS; pub const CR0_PE: i32 = 1; @@ -239,6 +240,8 @@ pub const MSR_PKG_C2_RESIDENCY: i32 = 0x60D; pub const IA32_KERNEL_GS_BASE: i32 = 0xC0000101u32 as i32; pub const MSR_AMD64_LS_CFG: i32 = 0xC0011020u32 as i32; pub const MSR_AMD64_DE_CFG: i32 = 0xC0011029u32 as i32; +pub const MSR_EFER: i32 = 0xC0000080u32 as i32; +pub const EFER_NXE: u64 = 1 << 11; pub const IA32_APIC_BASE_BSP: i32 = 1 << 8; pub const IA32_APIC_BASE_EXTD: i32 = 1 << 10; @@ -261,6 +264,7 @@ pub const TLB_NO_USER: i32 = 1 << 2; pub const TLB_IN_MAPPED_RANGE: i32 = 1 << 3; pub const TLB_GLOBAL: i32 = 1 << 4; pub const TLB_HAS_CODE: i32 = 1 << 5; +pub const TLB_NO_EXEC: i32 = 1 << 6; pub const IVT_SIZE: u32 = 0x400; pub const CPU_EXCEPTION_DE: i32 = 0; pub const CPU_EXCEPTION_DB: i32 = 1; @@ -1919,11 +1923,11 @@ pub unsafe fn readable_or_pagefault(addr: i32, size: i32) -> OrPageFault<()> { dbg_assert!(size > 0); let user = *cpl == 3; - translate_address(addr, false, user, false, true)?; + translate_address(addr, false, user, false, true, false)?; let end = addr + size - 1 & !0xFFF; if addr & !0xFFF != end & !0xFFF { - translate_address(end, false, user, false, true)?; + translate_address(end, false, user, false, true, false)?; } return Ok(()); @@ -1938,34 +1942,40 @@ pub unsafe fn writable_or_pagefault_cpl(other_cpl: u8, addr: i32, size: i32) -> dbg_assert!(size > 0); let user = other_cpl == 3; - translate_address(addr, true, user, false, true)?; + translate_address(addr, true, user, false, true, false)?; let end = addr + size - 1 & !0xFFF; if addr & !0xFFF != end & !0xFFF { - translate_address(end, true, user, false, true)?; + translate_address(end, true, user, false, true, false)?; } return Ok(()); } pub fn translate_address_read_no_side_effects(address: i32) -> OrPageFault { - unsafe { translate_address(address, false, *cpl == 3, false, false) } + unsafe { translate_address(address, false, *cpl == 3, false, false, false) } } pub fn translate_address_read(address: i32) -> OrPageFault { - unsafe { translate_address(address, false, *cpl == 3, false, true) } + unsafe { translate_address(address, false, *cpl == 3, false, true, false) } } pub unsafe fn translate_address_read_jit(address: i32) -> OrPageFault { - translate_address(address, false, *cpl == 3, true, true) + translate_address(address, false, *cpl == 3, true, true, false) +} +pub unsafe fn translate_address_fetch(address: i32) -> OrPageFault { + translate_address(address, false, *cpl == 3, false, true, true) +} +pub unsafe fn translate_address_fetch_jit(address: i32) -> OrPageFault { + translate_address(address, false, *cpl == 3, true, true, true) } pub unsafe fn translate_address_write(address: i32) -> OrPageFault { - translate_address(address, true, *cpl == 3, false, true) + translate_address(address, true, *cpl == 3, false, true, false) } pub unsafe fn translate_address_write_jit(address: i32, wasm_table_index: u16) -> OrPageFault { let mut entry = tlb_data[(address as u32 >> 12) as usize]; let user = *cpl == 3; if entry & (TLB_VALID | if user { TLB_NO_USER } else { 0 } | TLB_READONLY) != TLB_VALID { - entry = do_page_walk(address, true, user, true, true)?.get(); + entry = do_page_walk(address, true, user, true, true, false)?.get(); } let has_code = entry & TLB_HAS_CODE != 0; let phys_addr = (entry & !0xFFF ^ address) as u32 - memory::mem8 as u32; @@ -1989,10 +1999,10 @@ pub unsafe fn translate_address_write_jit(address: i32, wasm_table_index: u16) - } pub unsafe fn translate_address_system_read(address: i32) -> OrPageFault { - translate_address(address, false, false, false, true) + translate_address(address, false, false, false, true, false) } pub unsafe fn translate_address_system_write(address: i32) -> OrPageFault { - translate_address(address, true, false, false, true) + translate_address(address, true, false, false, true, false) } #[inline(always)] @@ -2002,15 +2012,25 @@ pub unsafe fn translate_address( user: bool, jit: bool, side_effects: bool, + is_instruction_fetch: bool, ) -> OrPageFault { let mut entry = tlb_data[(address as u32 >> 12) as usize]; if entry & (TLB_VALID | if user { TLB_NO_USER } else { 0 } - | if for_writing { TLB_READONLY } else { 0 }) + | if for_writing { TLB_READONLY } else { 0 } + | if is_instruction_fetch { TLB_NO_EXEC } else { 0 }) != TLB_VALID { - entry = do_page_walk(address, for_writing, user, jit, side_effects)?.get(); + entry = do_page_walk( + address, + for_writing, + user, + jit, + side_effects, + is_instruction_fetch, + )? + .get(); } Ok((entry & !0xFFF ^ address) as u32 - memory::mem8 as u32) } @@ -2019,7 +2039,7 @@ pub unsafe fn translate_address_write_and_can_skip_dirty(address: i32) -> OrPage let mut entry = tlb_data[(address as u32 >> 12) as usize]; let user = *cpl == 3; if entry & (TLB_VALID | if user { TLB_NO_USER } else { 0 } | TLB_READONLY) != TLB_VALID { - entry = do_page_walk(address, true, user, false, true)?.get(); + entry = do_page_walk(address, true, user, false, true, false)?.get(); } Ok(( (entry & !0xFFF ^ address) as u32 - memory::mem8 as u32, @@ -2037,7 +2057,7 @@ pub unsafe fn translate_address_write_and_can_skip_dirty(address: i32) -> OrPage // // Note that PAE entries are 64-bit, and can describe physical addresses over 32 // bits. However, since we support only 32-bit physical addresses, we require -// the high half of the entry to be 0. +// the high half of the entry to be 0, except for the NX bit when EFER.NXE is set. #[cold] pub unsafe fn do_page_walk( addr: i32, @@ -2045,6 +2065,7 @@ pub unsafe fn do_page_walk( user: bool, jit: bool, side_effects: bool, + is_instruction_fetch: bool, ) -> OrPageFault { let global; let mut allow_user = true; @@ -2053,6 +2074,8 @@ pub unsafe fn do_page_walk( let cr0 = *cr; let cr4 = *cr.offset(4); + let nxe = *efer & EFER_NXE != 0; + let mut no_exec = false; if cr0 & CR0_PG == 0 { // paging disabled @@ -2063,12 +2086,21 @@ pub unsafe fn do_page_walk( profiler::stat_increment(stat::TLB_MISS); let pae = cr4 & CR4_PAE != 0; + let pae_reserved_mask = 0x7FFF_FFFF_0000_0000 | if nxe { 0 } else { PAGE_TABLE_NX_MASK }; let (page_dir_addr, page_dir_entry) = if pae { let pdpt_entry = *reg_pdpte.offset(((addr as u32) >> 30) as isize); if pdpt_entry as i32 & PAGE_TABLE_PRESENT_MASK == 0 { if side_effects { - trigger_pagefault(addr, false, for_writing, user, jit); + trigger_pagefault( + addr, + false, + for_writing, + user, + jit, + is_instruction_fetch, + false, + ); } return Err(()); } @@ -2076,14 +2108,26 @@ pub unsafe fn do_page_walk( let page_dir_addr = (pdpt_entry as u32 & 0xFFFFF000) + ((((addr as u32) >> 21) & 0x1FF) << 3); let page_dir_entry = memory::read64s(page_dir_addr); - dbg_assert!( - page_dir_entry as u64 & 0x7FFF_FFFF_0000_0000 == 0, - "Unsupported: Page directory entry larger than 32 bits" - ); - dbg_assert!( - page_dir_entry & 0x8000_0000_0000_0000u64 as i64 == 0, - "Unsupported: NX bit" - ); + if page_dir_entry as i32 & PAGE_TABLE_PRESENT_MASK != 0 { + // bits 13 to 20 are reserved in a 2mb page directory entry + let reserved_mask = pae_reserved_mask + | if page_dir_entry as i32 & PAGE_TABLE_PSE_MASK != 0 { 0x1FE000 } else { 0 }; + if page_dir_entry as u64 & reserved_mask != 0 { + if side_effects { + trigger_pagefault( + addr, + true, + for_writing, + user, + jit, + is_instruction_fetch, + true, + ); + } + return Err(()); + } + no_exec = page_dir_entry as u64 & PAGE_TABLE_NX_MASK != 0; + } (page_dir_addr, page_dir_entry as i32) } @@ -2095,7 +2139,15 @@ pub unsafe fn do_page_walk( if page_dir_entry & PAGE_TABLE_PRESENT_MASK == 0 { if side_effects { - trigger_pagefault(addr, false, for_writing, user, jit); + trigger_pagefault( + addr, + false, + for_writing, + user, + jit, + is_instruction_fetch, + false, + ); } return Err(()); } @@ -2104,12 +2156,23 @@ pub unsafe fn do_page_walk( let mut allow_write = page_dir_entry & PAGE_TABLE_RW_MASK != 0; allow_user &= page_dir_entry & PAGE_TABLE_USER_MASK != 0; - if 0 != page_dir_entry & PAGE_TABLE_PSE_MASK && 0 != cr4 & CR4_PSE { + if 0 != page_dir_entry & PAGE_TABLE_PSE_MASK && (pae || 0 != cr4 & CR4_PSE) { // size bit is set - if for_writing && !allow_write && !kernel_write_override || user && !allow_user { + if for_writing && !allow_write && !kernel_write_override + || user && !allow_user + || is_instruction_fetch && no_exec + { if side_effects { - trigger_pagefault(addr, true, for_writing, user, jit); + trigger_pagefault( + addr, + true, + for_writing, + user, + jit, + is_instruction_fetch, + false, + ); } return Err(()); } @@ -2137,14 +2200,23 @@ pub unsafe fn do_page_walk( let page_table_addr = (page_dir_entry as u32 & 0xFFFFF000) + (((addr as u32 >> 12) & 0x1FF) << 3); let page_table_entry = memory::read64s(page_table_addr); - dbg_assert!( - page_table_entry as u64 & 0x7FFF_FFFF_0000_0000 == 0, - "Unsupported: Page table entry larger than 32 bits" - ); - dbg_assert!( - page_table_entry & 0x8000_0000_0000_0000u64 as i64 == 0, - "Unsupported: NX bit" - ); + if page_table_entry as i32 & PAGE_TABLE_PRESENT_MASK != 0 { + if page_table_entry as u64 & pae_reserved_mask != 0 { + if side_effects { + trigger_pagefault( + addr, + true, + for_writing, + user, + jit, + is_instruction_fetch, + true, + ); + } + return Err(()); + } + no_exec |= page_table_entry as u64 & PAGE_TABLE_NX_MASK != 0; + } (page_table_addr, page_table_entry as i32) } @@ -2162,9 +2234,18 @@ pub unsafe fn do_page_walk( if !present || for_writing && !allow_write && !kernel_write_override || user && !allow_user + || is_instruction_fetch && no_exec { if side_effects { - trigger_pagefault(addr, present, for_writing, user, jit); + trigger_pagefault( + addr, + present, + for_writing, + user, + jit, + is_instruction_fetch, + false, + ); } return Err(()); } @@ -2230,7 +2311,8 @@ pub unsafe fn do_page_walk( | if allow_user { 0 } else { TLB_NO_USER } | if is_in_mapped_range { TLB_IN_MAPPED_RANGE } else { 0 } | if global && 0 != cr4 & CR4_PGE { TLB_GLOBAL } else { 0 } - | if has_code { TLB_HAS_CODE } else { 0 }; + | if has_code { TLB_HAS_CODE } else { 0 } + | if no_exec { TLB_NO_EXEC } else { 0 }; let tlb_entry = (high + memory::mem8 as u32) as i32 ^ page << 12 | info_bits as i32; @@ -2240,7 +2322,8 @@ pub unsafe fn do_page_walk( // of memory accesses tlb_data[page as usize] = tlb_entry; - jit::update_tlb_code(Page::page_of(addr as u32), Page::page_of(high)); + let virt_page = Page::page_of(addr as u32); + jit::update_tlb_code(virt_page, Page::page_of(high)); } Ok(if DEBUG { @@ -2375,14 +2458,24 @@ pub unsafe fn exit_jit() { /// and finally calls exit_jit, which does the interrupt /// /// Non-jit resets the instruction pointer and does the PF interrupt directly -pub unsafe fn trigger_pagefault(addr: i32, present: bool, write: bool, user: bool, jit: bool) { +pub unsafe fn trigger_pagefault( + addr: i32, + present: bool, + write: bool, + user: bool, + jit: bool, + is_instruction_fetch: bool, + rsvd: bool, +) { if config::LOG_PAGE_FAULTS { dbg_log!( - "page fault{} w={} u={} p={} eip={:x} cr2={:x}", + "page fault{} w={}, x={}, u={} p={} r={} eip={:x} cr2={:x}", if jit { "jit" } else { "" }, write as i32, + is_instruction_fetch as i32, user as i32, present as i32, + rsvd as i32, *previous_ip, addr ); @@ -2394,7 +2487,13 @@ pub unsafe fn trigger_pagefault(addr: i32, present: bool, write: bool, user: boo let page = ((addr as u32) >> 12) as i32; clear_tlb_code(page); tlb_data[page as usize] = 0; - let error_code = (user as i32) << 2 | (write as i32) << 1 | present as i32; + let instruction_fetch = is_instruction_fetch + && (*cr.offset(4) & CR4_SMEP != 0 || *cr.offset(4) & CR4_PAE != 0 && *efer & EFER_NXE != 0); + let error_code = (instruction_fetch as i32) << 4 + | (rsvd as i32) << 3 + | (user as i32) << 2 + | (write as i32) << 1 + | present as i32; if jit { jit_exit_reason = JitExitReason::CpuException { code: CPU_EXCEPTION_PF, @@ -2403,6 +2502,11 @@ pub unsafe fn trigger_pagefault(addr: i32, present: bool, write: bool, user: boo } else { *instruction_pointer = *previous_ip; + if DEBUG { + if js::cpu_exception_hook(CPU_EXCEPTION_PF) { + return; + } + } call_interrupt_vector(CPU_EXCEPTION_PF, false, Some(error_code)); } } @@ -2480,7 +2584,7 @@ pub const DISABLE_EIP_TRANSLATION_OPTIMISATION: bool = false; pub unsafe fn read_imm8() -> OrPageFault { let eip = *instruction_pointer; if DISABLE_EIP_TRANSLATION_OPTIMISATION || 0 != eip & !0xFFF ^ *last_virt_eip { - *eip_phys = (translate_address_read(eip)? ^ eip as u32) as i32; + *eip_phys = (translate_address_fetch(eip)? ^ eip as u32) as i32; *last_virt_eip = eip & !0xFFF } dbg_assert!(!memory::in_mapped_range((*eip_phys ^ eip) as u32)); @@ -2842,6 +2946,7 @@ pub unsafe fn set_cr0(cr0: i32) { } if *cr.offset(4) & CR4_PAE != 0 + && cr0 & CR0_PG != 0 && old_cr0 & (CR0_CD | CR0_NW | CR0_PG) != cr0 & (CR0_CD | CR0_NW | CR0_PG) { load_pdpte(*cr.offset(3)) @@ -2857,7 +2962,9 @@ pub unsafe fn set_cr3(mut cr3: i32) { } if *cr.offset(4) & CR4_PAE != 0 { cr3 &= !0b1111; - load_pdpte(cr3); + if *cr.offset(0) & CR0_PG != 0 { + load_pdpte(cr3); + } } else { cr3 &= !0b111111100111; @@ -2869,13 +2976,16 @@ pub unsafe fn set_cr3(mut cr3: i32) { pub unsafe fn load_pdpte(cr3: i32) { dbg_assert!(cr3 & 0b1111 == 0); + // dbg_log!("load_pdpte cr3={:#010x}", cr3 as u32); for i in 0..4 { let mut pdpt_entry = memory::read64s(cr3 as u32 + 8 * i as u32) as u64; pdpt_entry &= !0b1110_0000_0000; - dbg_assert!(pdpt_entry & 0b11000 == 0, "TODO"); + // bits 3 (PWT) and 4 (PCD) are valid in PDPTE; ignore them (no cache emulation) + pdpt_entry &= !0b11000; dbg_assert!( pdpt_entry as u64 & 0xFFFF_FFFF_0000_0000 == 0, - "Unsupported: PDPT entry larger than 32 bits" + "Unsupported: PDPT entry larger than 32 bits: {:#018x}", + pdpt_entry ); if pdpt_entry as i32 & PAGE_TABLE_PRESENT_MASK != 0 { dbg_assert!( @@ -3178,7 +3288,7 @@ pub unsafe fn cycle_internal() { pub unsafe fn get_phys_eip() -> OrPageFault { let eip = *instruction_pointer; if 0 != eip & !0xFFF ^ *last_virt_eip { - *eip_phys = (translate_address_read(eip)? ^ eip as u32) as i32; + *eip_phys = (translate_address_fetch(eip)? ^ eip as u32) as i32; *last_virt_eip = eip & !0xFFF } let phys_addr = (*eip_phys ^ eip) as u32; @@ -3660,7 +3770,7 @@ pub unsafe fn safe_read128s_slow_jit(addr: i32, eip: i32) -> i32 { #[no_mangle] pub unsafe fn get_phys_eip_slow_jit(addr: i32) -> i32 { - match translate_address_read_jit(addr) { + match translate_address_fetch_jit(addr) { Err(()) => 1, Ok(addr_low) => { dbg_assert!(!memory::in_mapped_range(addr_low as u32)); // same assumption as in read_imm8 @@ -4541,7 +4651,7 @@ pub fn io_port_write32(port: i32, value: i32) { unsafe { js::io_port_write32(por #[no_mangle] #[cfg(debug_assertions)] pub unsafe fn check_page_switch(block_addr: u32, next_block_addr: u32) { - let x = translate_address_read_jit(*instruction_pointer); + let x = translate_address_fetch_jit(*instruction_pointer); if x != Ok(next_block_addr) { dbg_log!( "page switch from={:x} to={:x} prev_eip={:x} eip={:x} phys_eip={:x}", @@ -4632,6 +4742,7 @@ pub unsafe fn reset_cpu() { *last_op_size = 0; *pat = 0x0007_0406_0007_0406; + *efer = 0; set_tsc(0, 0); diff --git a/src/rust/cpu/global_pointers.rs b/src/rust/cpu/global_pointers.rs index d6d44a1c9d..0f8a0a5d3b 100644 --- a/src/rust/cpu/global_pointers.rs +++ b/src/rust/cpu/global_pointers.rs @@ -73,6 +73,7 @@ pub const tss_size_32: *mut bool = 1128 as *mut bool; pub const sse_scratch_register: *mut reg128 = 1136 as *mut reg128; pub const fpu_st: *mut F80 = 1152 as *mut F80; +pub const efer: *mut u64 = 1280 as *mut u64; pub const pat: *mut u64 = 1288 as *mut u64; pub fn get_reg32_offset(r: u32) -> u32 { diff --git a/src/rust/cpu/instructions_0f.rs b/src/rust/cpu/instructions_0f.rs index 4fbc2275d4..f475872429 100644 --- a/src/rust/cpu/instructions_0f.rs +++ b/src/rust/cpu/instructions_0f.rs @@ -811,12 +811,14 @@ pub unsafe fn instr_0F22(r: i32, creg: i32) { if 0 != (*cr.offset(4) ^ data) & (CR4_PGE | CR4_PSE | CR4_PAE) { full_clear_tlb(); } + let old_cr4 = *cr.offset(4); + *cr.offset(4) = data; if data & CR4_PAE != 0 - && 0 != (*cr.offset(4) ^ data) & (CR4_PGE | CR4_PSE | CR4_SMEP) + && 0 != (old_cr4 ^ data) & (CR4_PAE | CR4_PGE | CR4_PSE | CR4_SMEP) + && *cr.offset(0) & CR0_PG != 0 { load_pdpte(*cr.offset(3)); } - *cr.offset(4) = data; } }, _ => { @@ -1194,6 +1196,18 @@ pub unsafe fn instr_0F30() { } match index { + MSR_EFER => { + let value = (high as u64) << 32 | (low as u32) as u64; + dbg_assert!( + value & !EFER_NXE == 0, + "Unimplemented EFER bits: {:#x}", + value + ); + if value != *efer { + full_clear_tlb(); + } + *efer = value; + }, IA32_SYSENTER_CS => *sysenter_cs = low & 0xFFFF, IA32_SYSENTER_EIP => *sysenter_eip = low, IA32_SYSENTER_ESP => *sysenter_esp = low, @@ -1285,6 +1299,11 @@ pub unsafe fn instr_0F32() { let mut high = 0; match index { + MSR_EFER => { + let val = *efer; + low = val as i32; + high = (val >> 32) as i32; + }, IA32_SYSENTER_CS => low = *sysenter_cs, IA32_SYSENTER_EIP => low = *sysenter_eip, IA32_SYSENTER_ESP => low = *sysenter_esp, @@ -3265,10 +3284,10 @@ pub unsafe fn instr_0FA2() { if config::VMWARE_HYPERVISOR_PORT { ecx |= 1 << 31 }; // hypervisor - edx = (if true /* have fpu */ { 1 } else { 0 }) | // fpu - vme | 1 << 3 | 1 << 4 | 1 << 5 | 1 << 6 | // vme, pse, tsc, msr, pae - 1 << 8 | 1 << 11 | 1 << 13 | 1 << 15 | 1 << 16 | 1 << 19 | // cx8, sep, pge, cmov, pat, clflush - 1 << 23 | 1 << 24 | 1 << 25 | 1 << 26; // mmx, fxsr, sse1, sse2 + edx = (if true /* have fpu */ { 1 } else { 0 }) | // fpu + vme | 1 << 3 | 1 << 4 | 1 << 5 | 1 << 6 | // vme, pse, tsc, msr, pae + 1 << 8 | 1 << 11 | 1 << 13 | 1 << 15 | 1 << 16 | 1 << 19 | // cx8, sep, pge, cmov, pat, clflush + 1 << 23 | 1 << 24 | 1 << 25 | 1 << 26; // mmx, fxsr, sse1, sse2 if *acpi_enabled //&& this.apic_enabled[0]) @@ -3329,10 +3348,18 @@ pub unsafe fn instr_0FA2() { 0x80000000 => { // maximum supported extended level - eax = 5; + eax = 0x80000008u32 as i32; // other registers are reserved }, + 0x80000001 => { + let vme = 0 << 1; + edx = (if true /* have fpu */ { 1 } else { 0 }) | // fpu + vme | 1 << 3 | 1 << 4 | 1 << 5 | 1 << 6 | // vme, pse, tsc, msr, pae + 1 << 8 | 1 << 13 | 1 << 15 | 1 << 20 | // cx8, pge, cmov, nx + 1 << 23 | 1 << 24; // mmx, fxsr + }, + 0x40000000 => { // hypervisor if config::VMWARE_HYPERVISOR_PORT { @@ -3343,6 +3370,13 @@ pub unsafe fn instr_0FA2() { } }, + 0x80000008 => { + eax = 32 | 32 << 8; // physical and linear address widths + ebx = 0; + ecx = 0; + edx = 0; + }, + 0x15 => { eax = 1; // denominator ebx = 1; // numerator diff --git a/src/rust/jit.rs b/src/rust/jit.rs index fb2a388c86..574a78ac6a 100644 --- a/src/rust/jit.rs +++ b/src/rust/jit.rs @@ -1105,23 +1105,7 @@ pub fn codegen_finalize_finished( }, }; - for i in 0..unsafe { cpu::valid_tlb_entries_count } { - let page = unsafe { cpu::valid_tlb_entries[i as usize] }; - let entry = unsafe { cpu::tlb_data[page as usize] }; - if 0 != entry { - let tlb_physical_page = Page::of_u32( - (entry as u32 >> 12 ^ page as u32) - (unsafe { memory::mem8 } as u32 >> 12), - ); - if let Some(info) = pages.get(&tlb_physical_page) { - set_tlb_code( - Page::of_u32(page as u32), - wasm_table_index, - &info.entry_points, - state_flags, - ); - } - } - } + let compiled_pages: HashSet = pages.keys().copied().collect(); #[cfg(debug_assertions)] if CHECK_JIT_STATE_INVARIANTS { @@ -1132,6 +1116,7 @@ pub fn codegen_finalize_finished( let mut check_for_unused_wasm_table_index = HashSet::new(); for (page, mut info) in pages { + dbg_assert!(info.state_flags == state_flags); if let Some(old_entry) = ctx.pages.remove(&page) { info.hidden_wasm_table_indices .extend(old_entry.hidden_wasm_table_indices); @@ -1142,6 +1127,19 @@ pub fn codegen_finalize_finished( ctx.pages.insert(page, info); } + for i in 0..unsafe { cpu::valid_tlb_entries_count } { + let page = unsafe { cpu::valid_tlb_entries[i as usize] }; + let entry = unsafe { cpu::tlb_data[page as usize] }; + if 0 != entry { + let tlb_physical_page = Page::of_u32( + (entry as u32 >> 12 ^ page as u32) - (unsafe { memory::mem8 } as u32 >> 12), + ); + if compiled_pages.contains(&tlb_physical_page) { + update_tlb_code_ctx(&ctx, Page::of_u32(page as u32), tlb_physical_page); + } + } + } + let unused: Vec<&WasmTableIndex> = check_for_unused_wasm_table_index .iter() .filter(|&&i| ctx.pages.values().all(|page| page.wasm_table_index != i)) @@ -1161,32 +1159,32 @@ pub fn codegen_finalize_finished( } pub fn update_tlb_code(virt_page: Page, phys_page: Page) { - let ctx = get_jit_state(); + update_tlb_code_ctx(&get_jit_state(), virt_page, phys_page) +} - match ctx.pages.get(&phys_page) { - Some(PageInfo { - wasm_table_index, - entry_points, - state_flags, - hidden_wasm_table_indices: _, - }) => set_tlb_code(virt_page, *wasm_table_index, entry_points, *state_flags), - None => cpu::clear_tlb_code(virt_page.to_u32() as i32), +fn update_tlb_code_ctx(ctx: &JitState, virt_page: Page, phys_page: Page) { + if unsafe { cpu::tlb_data[virt_page.to_u32() as usize] } & (cpu::TLB_VALID | cpu::TLB_NO_EXEC) + != cpu::TLB_VALID + { + cpu::clear_tlb_code(virt_page.to_u32() as i32); + return; + } + + let info = match ctx.pages.get(&phys_page) { + Some(info) => info, + None => { + cpu::clear_tlb_code(virt_page.to_u32() as i32); + return; + }, }; -} -pub fn set_tlb_code( - virt_page: Page, - wasm_table_index: WasmTableIndex, - entries: &Vec<(u16, u16)>, - state_flags: CachedStateFlags, -) { let c = match unsafe { cpu::tlb_code[virt_page.to_u32() as usize] } { None => { let state_table = [u16::MAX; 0x1000]; unsafe { let mut c = NonNull::new_unchecked(Box::into_raw(Box::new(cpu::Code { - wasm_table_index, - state_flags, + wasm_table_index: info.wasm_table_index, + state_flags: info.state_flags, state_table, }))); cpu::tlb_code[virt_page.to_u32() as usize] = Some(c); @@ -1196,13 +1194,13 @@ pub fn set_tlb_code( Some(mut c) => unsafe { let c = c.as_mut(); c.state_table.fill(u16::MAX); - c.state_flags = state_flags; - c.wasm_table_index = wasm_table_index; + c.state_flags = info.state_flags; + c.wasm_table_index = info.wasm_table_index; c }, }; - for &(addr, state) in entries { + for &(addr, state) in &info.entry_points { dbg_assert!(state != u16::MAX); c.state_table[addr as usize] = state; } diff --git a/tests/api/state.js b/tests/api/state.js index 5017dcf42e..20a25bd828 100755 --- a/tests/api/state.js +++ b/tests/api/state.js @@ -66,15 +66,24 @@ async function run_test(name, config, done) await sleep(2000); console.log("Saving: %s", name); + + const expected_efer = Array.from(emulator.v86.cpu.efer); + emulator.v86.cpu.pat.set([0x07060504, 0x01000706]); const expected_pat = Array.from(emulator.v86.cpu.pat); + const state = await emulator.save_state(); await sleep(1000); console.log("Restoring: %s", name); + + emulator.v86.cpu.efer.fill(0); emulator.v86.cpu.pat.fill(0); + await emulator.restore_state(state); + + assert.deepEqual(Array.from(emulator.v86.cpu.efer), expected_efer, "EFER state"); assert.deepEqual(Array.from(emulator.v86.cpu.pat), expected_pat, "PAT state"); await emulator.wait_until_vga_screen_contains("~% "); diff --git a/tests/expect/tests/add8-mem.wast b/tests/expect/tests/add8-mem.wast index 7d38de2736..7c41d61134 100644 --- a/tests/expect/tests/add8-mem.wast +++ b/tests/expect/tests/add8-mem.wast @@ -94,7 +94,7 @@ (get_local $l9) (i32.const 12)) (i32.const 2)))) - (i32.const 4075)) + (i32.const 4011)) (i32.const 1)))) (br_if $B1 (i32.and diff --git a/tests/expect/tests/call-ret.wast b/tests/expect/tests/call-ret.wast index 2af1864e59..ba23920d20 100644 --- a/tests/expect/tests/call-ret.wast +++ b/tests/expect/tests/call-ret.wast @@ -172,7 +172,7 @@ (get_local $l11) (i32.const 12)) (i32.const 2)))) - (i32.const 4075)) + (i32.const 4011)) (i32.const 1)) (i32.le_s (i32.and @@ -263,7 +263,7 @@ (get_local $l9) (i32.const 12)) (i32.const 2)))) - (i32.const 4041)) + (i32.const 3977)) (i32.const 1)) (i32.le_s (i32.and diff --git a/tests/expect/tests/indirect-call.wast b/tests/expect/tests/indirect-call.wast index 551225baa6..af6a312154 100644 --- a/tests/expect/tests/indirect-call.wast +++ b/tests/expect/tests/indirect-call.wast @@ -168,7 +168,7 @@ (get_local $l9) (i32.const 12)) (i32.const 2)))) - (i32.const 4041)) + (i32.const 3977)) (i32.const 1)) (i32.le_s (i32.and @@ -222,7 +222,7 @@ (get_local $l12) (i32.const 12)) (i32.const 2)))) - (i32.const 4075)) + (i32.const 4011)) (i32.const 1)) (i32.le_s (i32.and diff --git a/tests/expect/tests/mem32r.wast b/tests/expect/tests/mem32r.wast index c30919542b..844d55d5fd 100644 --- a/tests/expect/tests/mem32r.wast +++ b/tests/expect/tests/mem32r.wast @@ -94,7 +94,7 @@ (get_local $l9) (i32.const 12)) (i32.const 2)))) - (i32.const 4041)) + (i32.const 3977)) (i32.const 1)) (i32.le_s (i32.and diff --git a/tests/expect/tests/mem32rmw.wast b/tests/expect/tests/mem32rmw.wast index 9136281bed..34fc2715a9 100644 --- a/tests/expect/tests/mem32rmw.wast +++ b/tests/expect/tests/mem32rmw.wast @@ -97,7 +97,7 @@ (get_local $l9) (i32.const 12)) (i32.const 2)))) - (i32.const 4075)) + (i32.const 4011)) (i32.const 1)) (i32.le_s (i32.and diff --git a/tests/expect/tests/mem32w.wast b/tests/expect/tests/mem32w.wast index 67b2456350..c27397a678 100644 --- a/tests/expect/tests/mem32w.wast +++ b/tests/expect/tests/mem32w.wast @@ -94,7 +94,7 @@ (get_local $l9) (i32.const 12)) (i32.const 2)))) - (i32.const 4075)) + (i32.const 4011)) (i32.const 1)) (i32.le_s (i32.and diff --git a/tests/expect/tests/mov-immoffs.wast b/tests/expect/tests/mov-immoffs.wast index 5a9080f97a..826b856ec2 100644 --- a/tests/expect/tests/mov-immoffs.wast +++ b/tests/expect/tests/mov-immoffs.wast @@ -92,7 +92,7 @@ (get_local $l9) (i32.const 12)) (i32.const 2)))) - (i32.const 4041)) + (i32.const 3977)) (i32.const 1)) (i32.le_s (i32.and diff --git a/tests/expect/tests/pop-esp.wast b/tests/expect/tests/pop-esp.wast index 14e1eca9a1..d3824991e3 100644 --- a/tests/expect/tests/pop-esp.wast +++ b/tests/expect/tests/pop-esp.wast @@ -183,7 +183,7 @@ (get_local $l10) (i32.const 12)) (i32.const 2)))) - (i32.const 4041)) + (i32.const 3977)) (i32.const 1)) (i32.le_s (i32.and @@ -224,7 +224,7 @@ (get_local $l9) (i32.const 12)) (i32.const 2)))) - (i32.const 4075)) + (i32.const 4011)) (i32.const 1)) (i32.le_s (i32.and diff --git a/tests/expect/tests/pop.wast b/tests/expect/tests/pop.wast index 5ec9892d65..3dfcef394a 100644 --- a/tests/expect/tests/pop.wast +++ b/tests/expect/tests/pop.wast @@ -83,7 +83,7 @@ (get_local $l9) (i32.const 12)) (i32.const 2)))) - (i32.const 4041)) + (i32.const 3977)) (i32.const 1)) (i32.le_s (i32.and diff --git a/tests/expect/tests/push.wast b/tests/expect/tests/push.wast index 5c16e433b2..d613abcc6c 100644 --- a/tests/expect/tests/push.wast +++ b/tests/expect/tests/push.wast @@ -86,7 +86,7 @@ (get_local $l10) (i32.const 12)) (i32.const 2)))) - (i32.const 4075)) + (i32.const 4011)) (i32.const 1)) (i32.le_s (i32.and diff --git a/tests/full/run.js b/tests/full/run.js index fd3e4e7206..e5e019c4d8 100755 --- a/tests/full/run.js +++ b/tests/full/run.js @@ -193,6 +193,17 @@ if(cluster.isPrimary) expect_mouse_registered: true, acpi: true, }, + { + name: "Windows 10", + skip_if_disk_image_missing: true, + hda: root_path + "/images/windows10.img", + memory_size: 1024 * 1024 * 1024, + timeout: 300, + expect_graphical_mode: true, + expect_graphical_size: [1024, 768], + expect_mouse_registered: true, + acpi: true, + }, { name: "Windows XP CD", skip_if_disk_image_missing: true, diff --git a/tests/kvm-unit-tests/README.md b/tests/kvm-unit-tests/README.md index 2df400ca23..c83e5f41de 100644 --- a/tests/kvm-unit-tests/README.md +++ b/tests/kvm-unit-tests/README.md @@ -13,8 +13,12 @@ make -C ../../build/libv86.js ./run.js x86/ioapic.flat ./run.js x86/apic.flat ./run.js x86/pae.flat +./run.mjs x86/nx.flat ``` +The NX test exercises PAE execute permissions, page-fault error codes and priority, +PDPTE reloads, and executable/NX aliases of the same physical page. + Tests can also be run in browser by going to `?profile=test-$name` (for example, `?profile=test-realmode`). diff --git a/tests/kvm-unit-tests/build.sh b/tests/kvm-unit-tests/build.sh index f6e5000f5f..4e22f4f221 100755 --- a/tests/kvm-unit-tests/build.sh +++ b/tests/kvm-unit-tests/build.sh @@ -2,4 +2,4 @@ set -euo pipefail cd "$(dirname "$0")" ./configure --arch=i386 -make CC="gcc -std=gnu11 -mno-sse -mno-sse2 -mno-mmx" x86/realmode.flat x86/taskswitch.flat x86/taskswitch2.flat x86/pat.flat +make CC="gcc -std=gnu11 -mno-sse -mno-sse2 -mno-mmx" x86/realmode.flat x86/taskswitch.flat x86/taskswitch2.flat x86/nx.flat x86/pat.flat diff --git a/tests/kvm-unit-tests/x86/Makefile.i386 b/tests/kvm-unit-tests/x86/Makefile.i386 index 548cab8dbc..f50e129ab4 100644 --- a/tests/kvm-unit-tests/x86/Makefile.i386 +++ b/tests/kvm-unit-tests/x86/Makefile.i386 @@ -4,7 +4,7 @@ ldarch = elf32-i386 cflatobjs += lib/x86/setjmp32.o -tests = $(TEST_DIR)/taskswitch.flat $(TEST_DIR)/taskswitch2.flat $(TEST_DIR)/pat.flat \ +tests = $(TEST_DIR)/taskswitch.flat $(TEST_DIR)/taskswitch2.flat $(TEST_DIR)/nx.flat $(TEST_DIR)/pat.flat \ $(TEST_DIR)/cmpxchg8b.flat diff --git a/tests/kvm-unit-tests/x86/nx.c b/tests/kvm-unit-tests/x86/nx.c new file mode 100644 index 0000000000..eb8d9b7b0f --- /dev/null +++ b/tests/kvm-unit-tests/x86/nx.c @@ -0,0 +1,196 @@ +/* NX permissions, page-fault priority, and cached executable aliases in PAE mode. */ +#include "libcflat.h" +#include "asm/page.h" +#include "processor.h" +#include "desc.h" +#include "msr.h" + +#define ALIAS 0x40000000UL +#define PRESENT (PT_PRESENT_MASK | PT_WRITABLE_MASK) +#define RESERVED (1ull << 62) +#define NO_FAULT (~0UL) + +static uint64_t pdpt[4] __attribute__((aligned(32))); +static uint64_t identity_pd[512] __attribute__((aligned(PAGE_SIZE))); +static uint64_t alias_pd[512] __attribute__((aligned(PAGE_SIZE))); +static uint64_t alias_pt[512] __attribute__((aligned(PAGE_SIZE))); +static uint32_t legacy_pd[1024] __attribute__((aligned(PAGE_SIZE))); +static unsigned char target[2 * PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); +static jmp_buf fault_env; +static volatile unsigned long fault_error, fault_address; + +static void finish_fault(void) +{ + longjmp(fault_env, 1); +} + +static void page_fault(struct ex_regs *regs) +{ + fault_error = regs->error_code; + fault_address = read_cr2(); + /* Return from the interrupt before restoring the test's stack. */ + regs->rip = (unsigned long)finish_fault; +} + +static void fetch(void *address) +{ + ((void (*)(void))address)(); +} + +static void read_data(void *address) +{ + (void)*(volatile unsigned char *)address; +} + +static void write_data(void *address) +{ + *(volatile unsigned char *)address = 0; +} + +static void read_float80(void *address) +{ + for (unsigned i = 0; i < 250000; i++) + asm volatile("fldt (%0); fstp %%st(0)" : : "r"(address) : "memory"); +} + +static void access_data_loop(void *address) +{ + volatile unsigned *value = address; + for (unsigned i = 0; i < 250000; i++) { + *value = i; + asm volatile("incl %0" : "+m"(*value)); + } + report("JIT data reads, writes and read-modify-writes preserve values", *value == 250000); +} + +static void check_access(const char *name, void (*access)(void *), + unsigned long address, unsigned long expected) +{ + fault_error = NO_FAULT; + fault_address = 0; + if (!setjmp(fault_env)) + access((void *)address); + report("%s (error=%lx, cr2=%lx)", + fault_error == expected && (expected == NO_FAULT || fault_address == address), + name, fault_error, fault_address); +} + +static void map_alias(uint64_t pde_flags, uint64_t pte_flags) +{ + alias_pd[0] = (uint32_t)alias_pt | pde_flags; + alias_pt[0] = (uint32_t)target | pte_flags; + alias_pt[1] = (uint32_t)(target + PAGE_SIZE) | pte_flags; + write_cr3((uint32_t)pdpt); +} + +static void setup_pae(void) +{ + for (unsigned i = 0; i < 512; i++) + identity_pd[i] = (uint64_t)i << 21 | PRESENT | PT_PAGE_SIZE_MASK; + pdpt[0] = (uint32_t)identity_pd | PT_PRESENT_MASK; + pdpt[1] = (uint32_t)alias_pd | PT_PRESENT_MASK; + write_cr0(read_cr0() & ~X86_CR0_PG); + write_cr4(read_cr4() | X86_CR4_PAE); + write_cr3((uint32_t)pdpt); + wrmsr(MSR_EFER, EFER_NX); + write_cr0(read_cr0() | X86_CR0_PE | X86_CR0_PG | X86_CR0_WP); +} + +static void test_nx(void) +{ + map_alias(PRESENT, PRESENT | PT64_NX_MASK); + check_access("NX PTE allows data reads", read_data, ALIAS, NO_FAULT); + check_access("NX PTE allows data writes", write_data, ALIAS + 32, NO_FAULT); + check_access("NX PTE allows repeated data accesses", access_data_loop, ALIAS + 32, NO_FAULT); + check_access("NX PTE allows 80-bit reads across pages", read_float80, + ALIAS + PAGE_SIZE - 4, NO_FAULT); + + /* Keep the NX alias in the TLB while the executable alias is compiled. */ + for (unsigned i = 0; i < 1000000; i++) + fetch(target); + check_access("compiled executable alias does not bypass NX", fetch, ALIAS, 0x11); + check_access("NX PTE blocks instruction fetch", fetch, ALIAS, 0x11); + + map_alias(PRESENT | PT64_NX_MASK, PRESENT); + check_access("NX PDE allows data reads", read_data, ALIAS, NO_FAULT); + check_access("NX PDE blocks instruction fetch", fetch, ALIAS, 0x11); + + map_alias(PRESENT | PT64_NX_MASK, 0); + check_access("absent PTE takes priority over NX PDE", fetch, ALIAS, 0x10); + map_alias(PRESENT | PT64_NX_MASK, RESERVED | PT64_NX_MASK); + check_access("reserved bits in absent PTE are ignored", fetch, ALIAS, 0x10); + map_alias(PRESENT | PT64_NX_MASK, PRESENT | RESERVED); + check_access("reserved PTE takes priority over NX PDE", fetch, ALIAS, 0x19); + map_alias(PRESENT | RESERVED, 0); + check_access("reserved PDE is checked before absent PTE", fetch, ALIAS, 0x19); + map_alias(RESERVED | PT64_NX_MASK, PRESENT); + check_access("reserved bits in absent PDE are ignored", fetch, ALIAS, 0x10); + + alias_pd[0] = ((uint32_t)target & ~(SZ_2M - 1)) | + PRESENT | PT_PAGE_SIZE_MASK | PT64_NX_MASK; + write_cr3((uint32_t)pdpt); + unsigned long huge_alias = ALIAS + ((uint32_t)target & (SZ_2M - 1)); + check_access("NX huge page allows data reads", read_data, huge_alias, NO_FAULT); + check_access("NX huge page blocks instruction fetch", fetch, huge_alias, 0x11); + + map_alias(PRESENT, PRESENT); + check_access("clearing NX allows execution again", fetch, ALIAS, NO_FAULT); + + wrmsr(MSR_EFER, 0); + report("EFER.NXE can be cleared", rdmsr(MSR_EFER) == 0); + map_alias(PRESENT, PRESENT | PT64_NX_MASK); + check_access("NX PTE is reserved without NXE on reads", read_data, ALIAS, 9); + check_access("NX PTE is reserved without NXE on writes", write_data, ALIAS, 11); + check_access("NX PTE is reserved without NXE on fetches", fetch, ALIAS, 9); + map_alias(PRESENT | PT64_NX_MASK, 0); + check_access("reserved NX PDE precedes absent PTE without NXE", fetch, ALIAS, 9); + map_alias(PRESENT, PT64_NX_MASK); + check_access("absent PTE without NXE clears the fetch error bit", fetch, ALIAS, 0); + wrmsr(MSR_EFER, EFER_NX); +} + +static void test_pdpte_reload(void) +{ + const unsigned flags[] = { 1 << 7 /* PGE */, X86_CR4_PSE }; + for (unsigned i = 0; i < ARRAY_SIZE(flags); i++) { + map_alias(PRESENT, PRESENT); + read_data((void *)ALIAS); + pdpt[1] = 0; + write_cr4(read_cr4() ^ flags[i]); + check_access("CR4 change reloads PDPTEs", read_data, ALIAS, 0); + pdpt[1] = (uint32_t)alias_pd | PT_PRESENT_MASK; + write_cr4(read_cr4() ^ flags[i]); + check_access("CR4 change restores PDPTE mapping", read_data, ALIAS, NO_FAULT); + } +} + +static void test_legacy_paging(void) +{ + for (unsigned i = 0; i < 16; i++) + legacy_pd[i] = i << 22 | PRESENT | PT_PAGE_SIZE_MASK; + write_cr0(read_cr0() & ~X86_CR0_PG); + write_cr4((read_cr4() & ~X86_CR4_PAE) | X86_CR4_PSE); + write_cr3((uint32_t)legacy_pd); + write_cr0(read_cr0() | X86_CR0_PG); + check_access("non-PAE fetch clears the fetch error bit even with NXE", fetch, ALIAS, 0); + wrmsr(MSR_EFER, 0); + check_access("non-PAE fetch clears the fetch error bit without NXE", fetch, ALIAS, 0); +} + +int main(void) +{ + report("CPUID exposes extended address widths", cpuid(0x80000000).a >= 0x80000008); + report("CPUID advertises NX", cpuid(0x80000001).d & (1 << 20)); + unsigned widths = cpuid(0x80000008).a; + report("CPUID reports physical and 32-bit linear address widths", + (widths & 0xFF) >= 32 && (widths >> 8 & 0xFF) == 32); + setup_idt(); + handle_exception(14, page_fault); + target[0] = 0xC3; /* ret */ + setup_pae(); + report("EFER.NXE can be set", rdmsr(MSR_EFER) == EFER_NX); + test_nx(); + test_pdpte_reload(); + test_legacy_paging(); + return report_summary(); +}